ExecutionBuilder.cpp revision 1da8fed77c5c296afa18f754ec3616e7f02a4cfd
1707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet/* 2707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * Copyright (C) 2017 The Android Open Source Project 3707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * 4707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * Licensed under the Apache License, Version 2.0 (the "License"); 5707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * you may not use this file except in compliance with the License. 6707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * You may obtain a copy of the License at 7707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * 8707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * http://www.apache.org/licenses/LICENSE-2.0 9707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * 10707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * Unless required by applicable law or agreed to in writing, software 11707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * distributed under the License is distributed on an "AS IS" BASIS, 12707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * See the License for the specific language governing permissions and 14707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * limitations under the License. 15707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet */ 16707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 173ced3cfd5b8f22b632c35f24e585c4847383b195David Gross#define LOG_TAG "ExecutionBuilder" 18707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 193ced3cfd5b8f22b632c35f24e585c4847383b195David Gross#include "ExecutionBuilder.h" 20707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 2183e24dc4706a5b7089881a55daf05b3924fab3b7David Gross#include "CompilationBuilder.h" 22707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "CpuExecutor.h" 23707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "HalInterfaces.h" 24707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "Manager.h" 25707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "ModelBuilder.h" 26707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 27689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler#include <mutex> 28389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet#include <thread> 29689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler#include <vector> 30689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 31707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouilletnamespace android { 32707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouilletnamespace nn { 33707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 34389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouilletint ModelArgumentInfo::setFromPointer(const Operand& operand, 35389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet const ANeuralNetworksOperandType* type, void* data, 36389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t length) { 37389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet int n = updateDimensionInfo(operand, type); 38389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 39389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return n; 40389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 4162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet if (data == nullptr) { 4262cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet if (length) { 4362cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet LOG(ERROR) << "Setting argument as having no value but non-zero length passed."; 4462cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 4562cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet } 4662cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet state = ModelArgumentInfo::HAS_NO_VALUE; 4762cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet } else { 4862cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet state = ModelArgumentInfo::POINTER; 4962cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet } 50389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet buffer = data; 5162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet locationAndLength = {.poolIndex = 0, .offset = 0, .length = length}; 52389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 53389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} 54389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet 55389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouilletint ModelArgumentInfo::setFromMemory(const Operand& operand, const ANeuralNetworksOperandType* type, 56389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t poolIndex, uint32_t offset, uint32_t length) { 57389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet int n = updateDimensionInfo(operand, type); 58389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 59389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return n; 60389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 61389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet state = ModelArgumentInfo::MEMORY; 6262cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet locationAndLength = {.poolIndex = poolIndex, .offset = offset, .length = length}; 63389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet buffer = nullptr; 64389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 65389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} 66389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet 678fb14e90ceb360adfbac0f708d27161b7c5b7fc5David Grossint ModelArgumentInfo::setFromTemporaryMemory(const Operand& operand, 688fb14e90ceb360adfbac0f708d27161b7c5b7fc5David Gross uint32_t poolIndex, uint32_t offset) { 694d83c52f52613585f7b86368be762b2857f7460fDavid Gross dimensions = operand.dimensions; 7096811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross state = ModelArgumentInfo::MEMORY; 714d83c52f52613585f7b86368be762b2857f7460fDavid Gross locationAndLength = 728fb14e90ceb360adfbac0f708d27161b7c5b7fc5David Gross {.poolIndex = poolIndex, .offset = offset, .length = sizeOfData(operand)}; 7396811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross buffer = nullptr; 7496811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross return ANEURALNETWORKS_NO_ERROR; 7596811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross} 7696811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 77389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouilletint ModelArgumentInfo::updateDimensionInfo(const Operand& operand, 78389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet const ANeuralNetworksOperandType* newType) { 79389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (newType == nullptr) { 8062cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet dimensions = hidl_vec<uint32_t>(); 81389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } else { 82d2d0c031c43e8e5aafc75e8a652d79bcc2aaca99Jean-Luc Brouillet uint32_t count = newType->dimensionCount; 83389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (static_cast<OperandType>(newType->type) != operand.type || 84389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet count != operand.dimensions.size()) { 853ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setInput/Output incompatible types"; 86389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 87389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 88389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet for (uint32_t i = 0; i < count; i++) { 8962cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet dimensions[i] = newType->dimensions[i]; 90389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 91389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 92389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 93389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} 94389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet 953ced3cfd5b8f22b632c35f24e585c4847383b195David GrossExecutionBuilder::ExecutionBuilder(const CompilationBuilder* compilation) : 9683e24dc4706a5b7089881a55daf05b3924fab3b7David Gross mModel(compilation->mModel), 971f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mPlan(&compilation->mPlan), 9883e24dc4706a5b7089881a55daf05b3924fab3b7David Gross mInputs(mModel->inputCount()), 991da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet mOutputs(mModel->outputCount()) { 100820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "ExecutionBuilder::ExecutionBuilder"; 101707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 102707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1033ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setInput(uint32_t index, const ANeuralNetworksOperandType* type, 104e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const void* buffer, size_t length) { 105707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mInputs.size()); 106707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (index >= count) { 1073ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setInput bad index " << index << " " << count; 108707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 109707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 110e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (type != nullptr) { 111e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet int n = validateOperandType(*type, "ANeuralNetworksExecution_setInput", false); 112e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 113e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return n; 114e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 115e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 116e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (length > 0xFFFFFFFF) { 117e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet LOG(ERROR) << "ANeuralNetworksExecution_setInput input exceeds max length " << length; 118e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 119e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 120e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet uint32_t l = static_cast<uint32_t>(length); 121389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return mInputs[index].setFromPointer(mModel->getInputOperand(index), type, 122e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const_cast<void*>(buffer), l); 123707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 124707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1253ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setInputFromMemory(uint32_t index, const ANeuralNetworksOperandType* type, 126e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const Memory* memory, size_t offset, size_t length) { 12796811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // Should be similar to StepExecutor::setInputOrOutputFromTemporaryMemory() 12896811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 1298b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mInputs.size()); 1308b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (index >= count) { 1313ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setInputFromMemory bad index " << index << " " 1328b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet << count; 1338b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 1348b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 135105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang if (!memory->validateSize(offset, length)) { 136105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang return ANEURALNETWORKS_BAD_DATA; 137105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang } 138e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet // TODO validate the rest 139389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t poolIndex = mMemories.add(memory); 140389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return mInputs[index].setFromMemory(mModel->getInputOperand(index), type, poolIndex, offset, 141389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet length); 142707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 143707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1443ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setOutput(uint32_t index, const ANeuralNetworksOperandType* type, void* buffer, 145e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet size_t length) { 146707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mOutputs.size()); 147707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (index >= count) { 1483ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setOutput bad index " << index << " " << count; 149707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 150707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 151e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (type != nullptr) { 152e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet int n = validateOperandType(*type, "ANeuralNetworksExecution_setOutput", false); 153e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 154e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return n; 155e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 156e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 157e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (length > 0xFFFFFFFF) { 158e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet LOG(ERROR) << "ANeuralNetworksExecution_setOutput input exceeds max length " << length; 159e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 160e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 161e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet uint32_t l = static_cast<uint32_t>(length); 162e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return mOutputs[index].setFromPointer(mModel->getOutputOperand(index), type, buffer, l); 163707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 164707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1653ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setOutputFromMemory(uint32_t index, const ANeuralNetworksOperandType* type, 166e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const Memory* memory, size_t offset, size_t length) { 16796811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // Should be similar to StepExecutor::setInputOrOutputFromTemporaryMemory() 16896811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 1698b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mOutputs.size()); 1708b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (index >= count) { 1713ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setOutputFromMemory bad index " << index << " " 1728b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet << count; 1738b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 1748b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 175105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang if (!memory->validateSize(offset, length)) { 176105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang return ANEURALNETWORKS_BAD_DATA; 177105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang } 178e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet // TODO validate the rest 179389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t poolIndex = mMemories.add(memory); 180389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return mOutputs[index].setFromMemory(mModel->getOutputOperand(index), type, poolIndex, offset, 181389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet length); 182707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 183707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1845e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// Attempt synchronous execution of full model on CPU. 1855e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// Ensure that executionCallback->notify() is called. 1865e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Grossstatic void cpuFallbackFull(const ExecutionBuilder* executionBuilder, 1875e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const sp<ExecutionCallback>& executionCallback) { 188820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "cpuFallbackFull"; 1895e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross StepExecutor executor(executionBuilder, executionBuilder->getModel(), 1905e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross nullptr /* no IDevice, so CPU */, 1915e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross nullptr /* no IPreparedModel */); 1925e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executor.mapInputsAndOutputsTrivially(); 1935e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross sp<ExecutionCallback> fallbackCallback; 1945e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (executor.startCompute(&fallbackCallback) != ANEURALNETWORKS_NO_ERROR) { 1955e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->notify(ErrorStatus::GENERAL_FAILURE); 1965e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 1975e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 1985e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross fallbackCallback->wait(); 1995e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->notify(fallbackCallback->getStatus()); 2005e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross} 2015e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross 2025e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// Attempt synchronous execution on CPU. 2035e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// (1) First, attempt to execute this step on CPU. If successful, 2045e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// return true. (Do not call executionCallback->notify().) 2055e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// (2) If unsuccessful, attempt to execute the full model on CPU, 2065e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// ensure that executionCallback->notify() is called, and return 2075e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// false. 2085e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Grossstatic bool cpuFallbackPartial(const ExecutionBuilder* executionBuilder, 2095e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const ExecutionPlan* plan, 2105e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross std::shared_ptr<ExecutionPlan::Controller> controller, 2115e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const sp<ExecutionCallback>& executionCallback) { 212820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "cpuFallbackPartial"; 2135e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross std::shared_ptr<StepExecutor> executor; 2145e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross int n = plan->fallback(controller, &executor); 2155e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (n != ANEURALNETWORKS_NO_ERROR || executor->isCpu()) { 2165e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross cpuFallbackFull(executionBuilder, executionCallback); 2175e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return false; 2185e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2195e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross sp<ExecutionCallback> fallbackCallback; 2205e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (executor->startComputeOnCpu(&fallbackCallback) != ANEURALNETWORKS_NO_ERROR) { 2215e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross cpuFallbackFull(executionBuilder, executionCallback); 2225e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return false; 2235e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2245e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross fallbackCallback->wait(); 2255e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (fallbackCallback->getStatus() != ErrorStatus::NONE) { 2265e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross cpuFallbackFull(executionBuilder, executionCallback); 2275e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return false; 2285e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2295e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return true; 2305e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross} 2315e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross 2325e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Grossstatic void asyncStartComputePartitioned(const ExecutionBuilder* executionBuilder, 2335e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const ExecutionPlan* plan, 234e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross std::shared_ptr<ExecutionPlan::Controller> controller, 2355e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross bool allowFallback, 2365e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const sp<ExecutionCallback>& executionCallback) { 237820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "ExecutionBuilder::startCompute (from plan, iteratively)"; 238e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross while (true) { 239e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross std::shared_ptr<StepExecutor> executor; 240820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "looking for next StepExecutor"; 241e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross int n = plan->next(controller, &executor); 2425e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (n != ANEURALNETWORKS_NO_ERROR) { 2435e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (allowFallback) { 2445e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross cpuFallbackFull(executionBuilder, executionCallback); 2455e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 2465e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->notify(ErrorStatus::GENERAL_FAILURE); 2475e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2485e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 2495e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2505e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (executor == nullptr) { 2515e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->notify(ErrorStatus::NONE); 252e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross return; 253e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross } 254e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross 255e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross sp<ExecutionCallback> stepCallback; 256e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross n = executor->startCompute(&stepCallback); 257e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross if (n != ANEURALNETWORKS_NO_ERROR) { 2585e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (allowFallback) { 2595e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (cpuFallbackPartial(executionBuilder, plan, controller, executionCallback)) { 2605e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Successfully executed one step on CPU. 2615e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross continue; 2625e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 2635e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Either successfully executed entire plan on 2645e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // CPU, or tried and failed to do so. 2655e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 2665e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2675e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 2685e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->notify(ErrorStatus::GENERAL_FAILURE); 2695e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 2705e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 271e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross } 272e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross stepCallback->wait(); 273e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross ErrorStatus status = stepCallback->getStatus(); 274e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross if (status != ErrorStatus::NONE) { 2755e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (allowFallback) { 2765e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (cpuFallbackPartial(executionBuilder, plan, controller, executionCallback)) { 2775e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Successfully executed one step on CPU. 2785e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross continue; 2795e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 2805e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Either successfully executed entire plan on 2815e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // CPU, or tried and failed to do so. 2825e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 2835e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2845e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 2855e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->notify(status); 2865e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 2875e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 288e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross } 289e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross } 290e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross} 291e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross 292033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint ExecutionBuilder::startCompute(sp<ExecutionCallback>* synchronizationCallback) { 293033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = nullptr; 294425b2594c76e934dfdbc93209253e3c189571149David Gross 295707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // TODO validate that we have full types for all inputs and outputs, 296707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // that the graph is not cyclic, 297f1817c663af4f22bc089ef82cd50df4186422c42Yang Ni 2988b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& p : mInputs) { 299389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (p.state == ModelArgumentInfo::UNSPECIFIED) { 3003ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_startCompute not all inputs specified"; 301707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 302707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 303707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 3048b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& p : mOutputs) { 305389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (p.state == ModelArgumentInfo::UNSPECIFIED) { 3063ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_startCompute not all outputs specified"; 307707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 308707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 309707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 310707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 3115e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross#ifndef DISABLE_PARTITIONED_EXECUTION 3125e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross { 3135e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // TODO: Remove the non-plan-based path once we've fully integrated ExecutionPlan 3145e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // with the compilation and execution phases of the NN API? Or retain that path 3155e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // as a fallback in the case of partitioning failure? 3165e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // 3175e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // TODO: Entire plan-based-path should run in an asynchronous thread -- 3185e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // take the asynchronous thread logic out of startComputeOnCpu() and use 3195e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // it to wrap the plan-based-path. 3205e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const uint32_t partitioning = DeviceManager::get()->getPartitioning(); 3215e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (partitioning > 0) { 3225e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const bool allowFallback = DeviceManager::partitioningAllowsFallback(partitioning); 3235e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross std::shared_ptr<ExecutionPlan::Controller> controller = mPlan->makeController(this); 3245e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (controller == nullptr) { 3255e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (!allowFallback) { 3265e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return ANEURALNETWORKS_OP_FAILED; 3275e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 3285e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 3295e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // TODO: use a thread pool 330b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 3315e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Prepare the callback for asynchronous execution. 3325e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // sp<ExecutionCallback> object is returned when the 3335e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // execution has been successfully launched, otherwise a 3345e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // nullptr is returned. The executionCallback is 3355e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // abstracted in the NN API as an "event". 3365e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross sp<ExecutionCallback> executionCallback = new ExecutionCallback(); 3375e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross std::thread thread(asyncStartComputePartitioned, this, mPlan, controller, 3385e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross allowFallback, 3395e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback); 3405e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->bind_thread(std::move(thread)); 3415e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross *synchronizationCallback = executionCallback; 3425e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return ANEURALNETWORKS_NO_ERROR; 3435e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 3441f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross } 3451f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross } 3465e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross#else 3475e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross { 3485e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Find a driver that can handle all the operations. 3495e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // TODO: Does not handle CPU fallback (which is tricky because 3505e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // StepExecutor::startCompute() is designed as 3515e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // asynchronous). 3525e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // TODO: Does not actually behave asynchronously (because 3535e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // StepExecutor::startCompute() isn't actually asynchronous 3545e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // on a device as opposed to a CPU). 3555e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross Model hidlModel; 3565e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross mModel->setHidlModel(&hidlModel); 3575e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const std::vector<std::shared_ptr<Device>>& devices = DeviceManager::get()->getDrivers(); 3585e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross for (const auto& device : devices) { 3595e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross hidl_vec<bool> supports; 360820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "Checking " << device->getName(); 3615e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross device->getSupportedOperations(hidlModel, &supports); 3625e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (std::find(supports.begin(), supports.end(), false) == supports.end()) { 363820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "ExecutionBuilder::startCompute (without plan) on " << device->getName(); 3645e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross StepExecutor executor(this, mModel, device->getInterface(), 3655e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross nullptr /* no IPreparedModel, so compile */); 3665e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executor.mapInputsAndOutputsTrivially(); 3675e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return executor.startCompute(synchronizationCallback); 3685e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 369ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet } 370ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet } 3715e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross#endif // DISABLE_PARTITIONED_EXECUTION 3725e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross 3735e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Run on the CPU. 374820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "ExecutionBuilder::startCompute (without plan) on CPU"; 375b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross StepExecutor executor(this, mModel, 376b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross nullptr /* no IDevice, so CPU */, 377b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross nullptr /* no IPreparedModel */); 378b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross executor.mapInputsAndOutputsTrivially(); 379033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return executor.startCompute(synchronizationCallback); 380707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 381707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 382707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet// Figures out how to place each of the input or outputs in a buffer. This just does the layout, 3838b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet// it does not copy data. Aligns each input a bit. 384b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Grossint StepExecutor::allocatePointerArgumentsToPool(std::vector<ModelArgumentInfo>* args, 385b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross Memory* memory) { 3868b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t nextPoolIndex = mMemories.size(); 387707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet int64_t total = 0; 3888b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& info : *args) { 3898b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (info.state == ModelArgumentInfo::POINTER) { 39062cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet DataLocation& loc = info.locationAndLength; 3918b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // TODO Good enough alignment? 3928b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet total += alignBytesNeeded(static_cast<uint32_t>(total), loc.length); 3938b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet loc.poolIndex = nextPoolIndex; 3948b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet loc.offset = static_cast<uint32_t>(total); 3958b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet total += loc.length; 3968b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 3978b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet }; 398707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (total > 0xFFFFFFFF) { 3993ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_startCompute Size of all inputs or outputs exceeds " 400707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet "2^32."; 401707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 402707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 4038b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet hidl_memory hidlMemory; 4048b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (total > 0) { 405389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet memory->create(total); // TODO check error 4068b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet mMemories.add(memory); 4078b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 408707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 409707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 410707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 41162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouilletstatic void setRequestArgumentArray(const std::vector<ModelArgumentInfo>& argumentInfos, 412a4e2ee81d015b5e26897d18b18dd50044faba62fJean-Luc Brouillet hidl_vec<RequestArgument>* ioInfos) { 4138b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet size_t count = argumentInfos.size(); 4148b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet ioInfos->resize(count); 4158b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (size_t i = 0; i < count; i++) { 41662cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet const auto& info = argumentInfos[i]; 41762cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet (*ioInfos)[i] = { .hasNoValue = info.state == ModelArgumentInfo::HAS_NO_VALUE, 41862cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet .location = info.locationAndLength, 41962cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet .dimensions = info.dimensions, 42062cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet }; 4218b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 4228b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet} 4238b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet 424b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David GrossStepExecutor::StepExecutor(const ExecutionBuilder* executionBuilder, 425b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross const ModelBuilder* model, 426b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross sp<IDevice> driver, sp<IPreparedModel> preparedModel) : 427b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mExecutionBuilder(executionBuilder), mModel(model), 428891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross mDriver(driver), mPreparedModel(preparedModel), 429891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross mInputs(model->inputCount()), mOutputs(model->outputCount()) {} 430b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 431b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Grossvoid StepExecutor::mapInputsAndOutputsTrivially() { 432b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mInputs = mExecutionBuilder->mInputs; 433b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mOutputs = mExecutionBuilder->mOutputs; 434b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mMemories = mExecutionBuilder->mMemories; 435b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross} 436b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 437891b10f7048c62a37a74c4b570be220089dfd55eDavid Grossvoid StepExecutor::mapInputOrOutput(const ModelArgumentInfo& builderInputOrOutput, 438891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross ModelArgumentInfo* executorInputOrOutput) { 439891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross *executorInputOrOutput = builderInputOrOutput; 440891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross switch (executorInputOrOutput->state) { 441891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross default: 442891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross nnAssert(!"unexpected ModelArgumentInfo::state"); 443891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross case ModelArgumentInfo::POINTER: 444891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross case ModelArgumentInfo::UNSPECIFIED: 445891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross break; 446891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross case ModelArgumentInfo::MEMORY: { 447891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const uint32_t builderPoolIndex = 44862cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet builderInputOrOutput.locationAndLength.poolIndex; 449891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const Memory* memory = mExecutionBuilder->mMemories[builderPoolIndex]; 450891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const uint32_t executorPoolIndex = mMemories.add(memory); 45162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet executorInputOrOutput->locationAndLength.poolIndex = 452891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross executorPoolIndex; 453891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross break; 454891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 455891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 456891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross} 457891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross 45896811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Grossint StepExecutor::setInputOrOutputFromTemporaryMemory(const Operand& inputOrOutputOperand, 4598fb14e90ceb360adfbac0f708d27161b7c5b7fc5David Gross const Memory* memory, uint32_t offset, 46096811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross ModelArgumentInfo* inputOrOutputInfo) { 46196811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // Should be similar to 46296811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // ExecutionBuilder::setInputFromMemory() 46396811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // ExecutionBuilder::setOutputFromMemory() 46496811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 46596811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross uint32_t poolIndex = mMemories.add(memory); 4668fb14e90ceb360adfbac0f708d27161b7c5b7fc5David Gross return inputOrOutputInfo->setFromTemporaryMemory(inputOrOutputOperand, poolIndex, offset); 46796811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross} 46896811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 469033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint StepExecutor::startCompute(sp<ExecutionCallback>* synchronizationCallback) { 470b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross if (mDriver == nullptr) { 471033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return startComputeOnCpu(synchronizationCallback); 472b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross } else { 473033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return startComputeOnDevice(synchronizationCallback); 474b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross } 475b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross} 476b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 477033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint StepExecutor::startComputeOnDevice(sp<ExecutionCallback>* synchronizationCallback) { 478b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross nnAssert(mDriver != nullptr); 479b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 480033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = nullptr; 481425b2594c76e934dfdbc93209253e3c189571149David Gross 482b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross // TODO: Remove the mPreparedModel == nullptr case once we've fully integrated 4831f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // ExecutionPlan with the compilation and execution phases of the NN API 484b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross if (mPreparedModel == nullptr) { 4851f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross Model model; 4861f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mModel->setHidlModel(&model); 4871f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross 4881f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // TODO Dangerous! In async, the model will outlive it here. Safe for now 489033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback(); 490033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler Return<ErrorStatus> prepareLaunchStatus = 491033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler mDriver->prepareModel(model, preparedModelCallback); 492033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler if (!prepareLaunchStatus.isOk() || prepareLaunchStatus != ErrorStatus::NONE) { 493033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return ANEURALNETWORKS_OP_FAILED; 494033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler } 4955f916fc9a7ae95f172492bdfe5344c37beff3a6fMichael Butler 496033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // Immediately synchronize with callback object for now 4971f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // TODO: change to asynchronous later 498033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler preparedModelCallback->wait(); 499033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); 500033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler mPreparedModel = preparedModelCallback->getPreparedModel(); 501033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler if (prepareReturnStatus != ErrorStatus::NONE || mPreparedModel == nullptr) { 5021f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross return ANEURALNETWORKS_OP_FAILED; 5031f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross } 504707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 505707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 506e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // We separate the input & output pools so that we reduce the copying done if we 507e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // do an eventual remoting (hidl_memory->update()). We could also use it to set 508e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // protection on read only memory but that's not currently done. 509e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross Memory inputPointerArguments; 510e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross Memory outputPointerArguments; 511e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross 5128b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // Layout the input and output data 513e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross int n = allocatePointerArgumentsToPool(&mInputs, &inputPointerArguments); 514707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 515707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return n; 516707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 517e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross n = allocatePointerArgumentsToPool(&mOutputs, &outputPointerArguments); 518707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 519707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return n; 520707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 521707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 5228b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // Copy the input data that was specified via a pointer. 523e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // inputPointerArguments.update(); 5248b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& info : mInputs) { 5258b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (info.state == ModelArgumentInfo::POINTER) { 52662cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet DataLocation& loc = info.locationAndLength; 5272150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet uint8_t* data = nullptr; 528e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross int n = inputPointerArguments.getPointer(&data); 5292150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 5302150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet return n; 5312150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet } 5328b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet memcpy(data + loc.offset, info.buffer, loc.length); 5338b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 534707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 535e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // TODO: Add inputPointerArguments.commit() and .update() at all the right places 536707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 537707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet Request request; 53862cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mInputs, &request.inputs); 53962cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mOutputs, &request.outputs); 5408b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = mMemories.size(); 5418b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet request.pools.resize(count); 5428b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (uint32_t i = 0; i < count; i++) { 5438b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet request.pools[i] = mMemories[i]->getHidlMemory(); 5448b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 545707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 546033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // Prepare the callback for asynchronous execution. sp<ExecutionCallback> 547033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // object is returned when the execution has been successfully launched, 548033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // otherwise a nullptr is returned. The executionCallback is abstracted in 549033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // the NN API as an "event". 550033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // 551033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // The sp is used for ref-counting purposes. Without it, the HIDL service 552033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // could attempt to communicate with a dead callback object. 553689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // 554033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // TODO: Explain the "dead callback" problem further, either here or 555689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // in the design document. 556033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler sp<ExecutionCallback> executionCallback = new ExecutionCallback(); 557689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 558820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "Before mPreparedModel->execute() " << toString(request); 5593ced3cfd5b8f22b632c35f24e585c4847383b195David Gross // Execute. 560033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // TODO: What happens to the Callback if the service dies abnormally 561033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // -- won't that keep the Callback live forever, because the service 562689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // never has the opportunity to bump the reference count down? Or 563689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // maybe the HIDL infrastructure handles this magically? At worst, 564033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // it seems like this is a small memory leak, if the Callback stays 565689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // alive forever. 566033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler if (mPreparedModel->execute(request, executionCallback) != ErrorStatus::NONE) { 567820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "**Execute failed**"; 568707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_OP_FAILED; 569707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 570707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 571689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: Remove this synchronization point when the block of code below is 572689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // removed. 573033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler executionCallback->wait(); 574033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler Return<ErrorStatus> executionStatus = executionCallback->getStatus(); 575033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler if (!executionStatus.isOk() || executionStatus != ErrorStatus::NONE) { 576820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "**Execute async failed**"; 577689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler return ANEURALNETWORKS_OP_FAILED; 578689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler } 579689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 580707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // Copy the output data from shared memory to the output buffers. 581689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: Move this block of code somewhere else. It should not be in the 582689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // startCompute function. 583689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: outputMemory->update(); outputMemory->commit() 5848b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& info : mOutputs) { 5858b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (info.state == ModelArgumentInfo::POINTER) { 58662cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet DataLocation& loc = info.locationAndLength; 5872150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet uint8_t* data = nullptr; 588e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross int n = outputPointerArguments.getPointer(&data); 5892150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 5902150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet return n; 5912150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet } 5928b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet memcpy(info.buffer, data + loc.offset, loc.length); 5938b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 594707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 595820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "StepExecutor::startComputeOnDevice completed"; 596707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 597033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = executionCallback; 598707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 599707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 600707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 601689d892203c06c66c7bb2e374462a8434e40b75fMichael Butlerstatic void asyncStartComputeOnCpu(const Model& model, const Request& request, 6021da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet const std::vector<RunTimePoolInfo>& modelPoolInfos, 6031da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet const std::vector<RunTimePoolInfo>& requestPoolInfos, 604033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler const sp<IExecutionCallback>& executionCallback) { 605689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler CpuExecutor executor; 6061da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet int err = executor.run(model, request, modelPoolInfos, requestPoolInfos); 6075f916fc9a7ae95f172492bdfe5344c37beff3a6fMichael Butler ErrorStatus status = err == ANEURALNETWORKS_NO_ERROR ? 6085f916fc9a7ae95f172492bdfe5344c37beff3a6fMichael Butler ErrorStatus::NONE : ErrorStatus::GENERAL_FAILURE; 609033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler executionCallback->notify(status); 610689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler} 611689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 612033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint StepExecutor::startComputeOnCpu(sp<ExecutionCallback>* synchronizationCallback) { 613707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // TODO: use a thread pool 614689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 6151f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross Model model; 6161f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mModel->setHidlModel(&model); 6171f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross 618033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // Prepare the callback for asynchronous execution. sp<ExecutionCallback> 619033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // object is returned when the execution has been successfully launched, 620033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // otherwise a nullptr is returned. The executionCallback is abstracted in 621033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // the NN API as an "event". 622033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler sp<ExecutionCallback> executionCallback = new ExecutionCallback(); 623033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = nullptr; 624707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 6251da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet std::vector<RunTimePoolInfo> modelPoolInfos; 6261da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet if (!setRunTimePoolInfosFromHidlMemories(&modelPoolInfos, model.pools)) { 6271da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet return ANEURALNETWORKS_UNMAPPABLE; 6281da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet } 6291da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet 6301da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet std::vector<RunTimePoolInfo> requestPoolInfos; 6318b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = mMemories.size(); 6321da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet requestPoolInfos.resize(count); 6338b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (uint32_t i = 0; i < count; i++) { 6348b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet const Memory* mem = mMemories[i]; 6351da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet if (!requestPoolInfos[i].set(mem->getHidlMemory())) { 6361da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet return ANEURALNETWORKS_UNMAPPABLE; 6371da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet } 6388b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 6398b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // Create as many pools as there are input / output. 6401da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet auto fixPointerArguments = [&requestPoolInfos](std::vector<ModelArgumentInfo>& argumentInfos) { 6418b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (ModelArgumentInfo& argumentInfo : argumentInfos) { 6428b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (argumentInfo.state == ModelArgumentInfo::POINTER) { 643389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet RunTimePoolInfo runTimeInfo = { 644389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet .buffer = static_cast<uint8_t*>(argumentInfo.buffer)}; 64562cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet argumentInfo.locationAndLength.poolIndex = 6461da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet static_cast<uint32_t>(requestPoolInfos.size()); 64762cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet argumentInfo.locationAndLength.offset = 0; 6481da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet requestPoolInfos.push_back(runTimeInfo); 6498b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 6508b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 6518b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet }; 6528b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet fixPointerArguments(mInputs); 6538b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet fixPointerArguments(mOutputs); 654707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 6558b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet Request request; 65662cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mInputs, &request.inputs); 65762cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mOutputs, &request.outputs); 658707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 659689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: should model be moved with a std::cref? 660689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler std::thread thread(asyncStartComputeOnCpu, model, std::move(request), 6611da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet std::move(modelPoolInfos), std::move(requestPoolInfos), 6621da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet executionCallback); 663033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler executionCallback->bind_thread(std::move(thread)); 664689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 665033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = executionCallback; 666689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler return ANEURALNETWORKS_NO_ERROR; 667707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 668707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 669389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} // namespace nn 670389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} // namespace android 671