ExecutionBuilder.cpp revision 43d018d1a66872031568857fa369024c9640a085
1707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet/* 2707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * Copyright (C) 2017 The Android Open Source Project 3707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * 4707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * Licensed under the Apache License, Version 2.0 (the "License"); 5707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * you may not use this file except in compliance with the License. 6707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * You may obtain a copy of the License at 7707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * 8707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * http://www.apache.org/licenses/LICENSE-2.0 9707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * 10707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * Unless required by applicable law or agreed to in writing, software 11707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * distributed under the License is distributed on an "AS IS" BASIS, 12707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * See the License for the specific language governing permissions and 14707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * limitations under the License. 15707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet */ 16707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 173ced3cfd5b8f22b632c35f24e585c4847383b195David Gross#define LOG_TAG "ExecutionBuilder" 18707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 193ced3cfd5b8f22b632c35f24e585c4847383b195David Gross#include "ExecutionBuilder.h" 20707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 2183e24dc4706a5b7089881a55daf05b3924fab3b7David Gross#include "CompilationBuilder.h" 22707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "CpuExecutor.h" 23707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "HalInterfaces.h" 24707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "Manager.h" 25707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "ModelBuilder.h" 26707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 27689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler#include <mutex> 28389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet#include <thread> 29689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler#include <vector> 30689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 31707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouilletnamespace android { 32707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouilletnamespace nn { 33707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 34389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouilletint ModelArgumentInfo::setFromPointer(const Operand& operand, 35389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet const ANeuralNetworksOperandType* type, void* data, 36389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t length) { 3743d018d1a66872031568857fa369024c9640a085Miao Wang if ((data == nullptr) != (length == 0)) { 3843d018d1a66872031568857fa369024c9640a085Miao Wang LOG(ERROR) << "Data pointer must be nullptr if and only if length is zero (data = " 3943d018d1a66872031568857fa369024c9640a085Miao Wang << data << ", length = " << length << ")"; 4043d018d1a66872031568857fa369024c9640a085Miao Wang return ANEURALNETWORKS_BAD_DATA; 41389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 4262cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet if (data == nullptr) { 4362cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet state = ModelArgumentInfo::HAS_NO_VALUE; 4462cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet } else { 4543d018d1a66872031568857fa369024c9640a085Miao Wang int n = updateDimensionInfo(operand, type); 4643d018d1a66872031568857fa369024c9640a085Miao Wang if (n != ANEURALNETWORKS_NO_ERROR) { 4743d018d1a66872031568857fa369024c9640a085Miao Wang return n; 4843d018d1a66872031568857fa369024c9640a085Miao Wang } 4943d018d1a66872031568857fa369024c9640a085Miao Wang uint32_t neededLength = sizeOfData(operand.type, dimensions); 5043d018d1a66872031568857fa369024c9640a085Miao Wang if (operand.type != OperandType::OEM && neededLength != length) { 5143d018d1a66872031568857fa369024c9640a085Miao Wang LOG(ERROR) << "Setting argument with invalid length: " << length 5243d018d1a66872031568857fa369024c9640a085Miao Wang << ", expected length: " << neededLength; 5343d018d1a66872031568857fa369024c9640a085Miao Wang return ANEURALNETWORKS_BAD_DATA; 5443d018d1a66872031568857fa369024c9640a085Miao Wang } 5562cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet state = ModelArgumentInfo::POINTER; 5662cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet } 57389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet buffer = data; 5862cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet locationAndLength = {.poolIndex = 0, .offset = 0, .length = length}; 59389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 60389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} 61389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet 62389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouilletint ModelArgumentInfo::setFromMemory(const Operand& operand, const ANeuralNetworksOperandType* type, 63389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t poolIndex, uint32_t offset, uint32_t length) { 64389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet int n = updateDimensionInfo(operand, type); 65389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 66389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return n; 67389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 6843d018d1a66872031568857fa369024c9640a085Miao Wang uint32_t neededLength = sizeOfData(operand.type, dimensions); 6943d018d1a66872031568857fa369024c9640a085Miao Wang if (operand.type != OperandType::OEM && neededLength != length) { 7043d018d1a66872031568857fa369024c9640a085Miao Wang LOG(ERROR) << "Setting argument with invalid length: " << length 7143d018d1a66872031568857fa369024c9640a085Miao Wang << ", expected length: " << neededLength; 7243d018d1a66872031568857fa369024c9640a085Miao Wang return ANEURALNETWORKS_BAD_DATA; 7343d018d1a66872031568857fa369024c9640a085Miao Wang } 7443d018d1a66872031568857fa369024c9640a085Miao Wang 75389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet state = ModelArgumentInfo::MEMORY; 7662cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet locationAndLength = {.poolIndex = poolIndex, .offset = offset, .length = length}; 77389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet buffer = nullptr; 78389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 79389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} 80389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet 818fb14e90ceb360adfbac0f708d27161b7c5b7fc5David Grossint ModelArgumentInfo::setFromTemporaryMemory(const Operand& operand, 828fb14e90ceb360adfbac0f708d27161b7c5b7fc5David Gross uint32_t poolIndex, uint32_t offset) { 8343d018d1a66872031568857fa369024c9640a085Miao Wang int n = updateDimensionInfo(operand, nullptr); 8443d018d1a66872031568857fa369024c9640a085Miao Wang if (n != ANEURALNETWORKS_NO_ERROR) { 8543d018d1a66872031568857fa369024c9640a085Miao Wang return n; 8643d018d1a66872031568857fa369024c9640a085Miao Wang } 8796811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross state = ModelArgumentInfo::MEMORY; 884d83c52f52613585f7b86368be762b2857f7460fDavid Gross locationAndLength = 898fb14e90ceb360adfbac0f708d27161b7c5b7fc5David Gross {.poolIndex = poolIndex, .offset = offset, .length = sizeOfData(operand)}; 9096811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross buffer = nullptr; 9196811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross return ANEURALNETWORKS_NO_ERROR; 9296811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross} 9396811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 94389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouilletint ModelArgumentInfo::updateDimensionInfo(const Operand& operand, 95389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet const ANeuralNetworksOperandType* newType) { 9643d018d1a66872031568857fa369024c9640a085Miao Wang nnAssert(dimensions.empty()); 97389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (newType == nullptr) { 9843d018d1a66872031568857fa369024c9640a085Miao Wang for (auto i : operand.dimensions) { 9943d018d1a66872031568857fa369024c9640a085Miao Wang if (i == 0) { 10043d018d1a66872031568857fa369024c9640a085Miao Wang LOG(ERROR) << "Setting input/output with unspecified dimensions"; 10143d018d1a66872031568857fa369024c9640a085Miao Wang return ANEURALNETWORKS_BAD_DATA; 10243d018d1a66872031568857fa369024c9640a085Miao Wang } 10343d018d1a66872031568857fa369024c9640a085Miao Wang } 10443d018d1a66872031568857fa369024c9640a085Miao Wang dimensions = operand.dimensions; 105389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } else { 106d2d0c031c43e8e5aafc75e8a652d79bcc2aaca99Jean-Luc Brouillet uint32_t count = newType->dimensionCount; 107389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (static_cast<OperandType>(newType->type) != operand.type || 108389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet count != operand.dimensions.size()) { 10943d018d1a66872031568857fa369024c9640a085Miao Wang LOG(ERROR) << "Setting input/output with incompatible types"; 110389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 111389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 11246a54e8a58e39c2fcb3448de04be76d95e744618Chenjie Luo 11346a54e8a58e39c2fcb3448de04be76d95e744618Chenjie Luo dimensions = hidl_vec<uint32_t>(count); 114389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet for (uint32_t i = 0; i < count; i++) { 11543d018d1a66872031568857fa369024c9640a085Miao Wang if (operand.dimensions[i] != 0 && operand.dimensions[i] != newType->dimensions[i]) { 11643d018d1a66872031568857fa369024c9640a085Miao Wang LOG(ERROR) << "Overriding a fully specified dimension is disallowed"; 11743d018d1a66872031568857fa369024c9640a085Miao Wang return ANEURALNETWORKS_BAD_DATA; 11843d018d1a66872031568857fa369024c9640a085Miao Wang } else { 11943d018d1a66872031568857fa369024c9640a085Miao Wang dimensions[i] = newType->dimensions[i]; 12043d018d1a66872031568857fa369024c9640a085Miao Wang } 121389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 122389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 123389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 124389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} 125389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet 1263ced3cfd5b8f22b632c35f24e585c4847383b195David GrossExecutionBuilder::ExecutionBuilder(const CompilationBuilder* compilation) : 12783e24dc4706a5b7089881a55daf05b3924fab3b7David Gross mModel(compilation->mModel), 1281f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mPlan(&compilation->mPlan), 129c2f1c1198c84f5a75fc2305935155f33b8ff5db2David Gross mPartitioning(compilation->mPartitioning), 13083e24dc4706a5b7089881a55daf05b3924fab3b7David Gross mInputs(mModel->inputCount()), 1311da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet mOutputs(mModel->outputCount()) { 132820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "ExecutionBuilder::ExecutionBuilder"; 133707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 134707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1353ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setInput(uint32_t index, const ANeuralNetworksOperandType* type, 136e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const void* buffer, size_t length) { 137707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mInputs.size()); 138707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (index >= count) { 1393ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setInput bad index " << index << " " << count; 140707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 141707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 142e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (type != nullptr) { 143e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet int n = validateOperandType(*type, "ANeuralNetworksExecution_setInput", false); 144e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 145e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return n; 146e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 147e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 148e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (length > 0xFFFFFFFF) { 149e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet LOG(ERROR) << "ANeuralNetworksExecution_setInput input exceeds max length " << length; 150e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 151e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 152e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet uint32_t l = static_cast<uint32_t>(length); 153389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return mInputs[index].setFromPointer(mModel->getInputOperand(index), type, 154e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const_cast<void*>(buffer), l); 155707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 156707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1573ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setInputFromMemory(uint32_t index, const ANeuralNetworksOperandType* type, 158e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const Memory* memory, size_t offset, size_t length) { 15996811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // Should be similar to StepExecutor::setInputOrOutputFromTemporaryMemory() 16096811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 1618b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mInputs.size()); 1628b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (index >= count) { 1633ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setInputFromMemory bad index " << index << " " 1648b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet << count; 1658b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 1668b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 167105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang if (!memory->validateSize(offset, length)) { 168105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang return ANEURALNETWORKS_BAD_DATA; 169105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang } 170e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet // TODO validate the rest 171389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t poolIndex = mMemories.add(memory); 172389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return mInputs[index].setFromMemory(mModel->getInputOperand(index), type, poolIndex, offset, 173389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet length); 174707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 175707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1763ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setOutput(uint32_t index, const ANeuralNetworksOperandType* type, void* buffer, 177e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet size_t length) { 178707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mOutputs.size()); 179707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (index >= count) { 1803ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setOutput bad index " << index << " " << count; 181707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 182707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 183e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (type != nullptr) { 184e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet int n = validateOperandType(*type, "ANeuralNetworksExecution_setOutput", false); 185e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 186e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return n; 187e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 188e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 189e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (length > 0xFFFFFFFF) { 190e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet LOG(ERROR) << "ANeuralNetworksExecution_setOutput input exceeds max length " << length; 191e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 192e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 193e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet uint32_t l = static_cast<uint32_t>(length); 194e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return mOutputs[index].setFromPointer(mModel->getOutputOperand(index), type, buffer, l); 195707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 196707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1973ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setOutputFromMemory(uint32_t index, const ANeuralNetworksOperandType* type, 198e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const Memory* memory, size_t offset, size_t length) { 19996811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // Should be similar to StepExecutor::setInputOrOutputFromTemporaryMemory() 20096811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 2018b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mOutputs.size()); 2028b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (index >= count) { 2033ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setOutputFromMemory bad index " << index << " " 2048b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet << count; 2058b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 2068b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 207105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang if (!memory->validateSize(offset, length)) { 208105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang return ANEURALNETWORKS_BAD_DATA; 209105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang } 210e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet // TODO validate the rest 211389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t poolIndex = mMemories.add(memory); 212389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return mOutputs[index].setFromMemory(mModel->getOutputOperand(index), type, poolIndex, offset, 213389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet length); 214707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 215707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 2165e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// Attempt synchronous execution of full model on CPU. 2175e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// Ensure that executionCallback->notify() is called. 2185e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Grossstatic void cpuFallbackFull(const ExecutionBuilder* executionBuilder, 2195e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const sp<ExecutionCallback>& executionCallback) { 220820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "cpuFallbackFull"; 2215e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross StepExecutor executor(executionBuilder, executionBuilder->getModel(), 22275886e77f9ca074173a49283b5c0a8c182d98977Michael Butler nullptr /* no VersionedIDevice, so CPU */, 2235e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross nullptr /* no IPreparedModel */); 2245e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executor.mapInputsAndOutputsTrivially(); 2255e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross sp<ExecutionCallback> fallbackCallback; 2265e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (executor.startCompute(&fallbackCallback) != ANEURALNETWORKS_NO_ERROR) { 2275e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->notify(ErrorStatus::GENERAL_FAILURE); 2285e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 2295e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2305e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross fallbackCallback->wait(); 2315e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->notify(fallbackCallback->getStatus()); 2325e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross} 2335e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross 2345e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// Attempt synchronous execution on CPU. 2355e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// (1) First, attempt to execute this step on CPU. If successful, 2365e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// return true. (Do not call executionCallback->notify().) 2375e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// (2) If unsuccessful, attempt to execute the full model on CPU, 2385e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// ensure that executionCallback->notify() is called, and return 2395e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// false. 2405e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Grossstatic bool cpuFallbackPartial(const ExecutionBuilder* executionBuilder, 2415e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const ExecutionPlan* plan, 2425e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross std::shared_ptr<ExecutionPlan::Controller> controller, 2435e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const sp<ExecutionCallback>& executionCallback) { 244820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "cpuFallbackPartial"; 2455e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross std::shared_ptr<StepExecutor> executor; 2465e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross int n = plan->fallback(controller, &executor); 2475e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (n != ANEURALNETWORKS_NO_ERROR || executor->isCpu()) { 2485e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross cpuFallbackFull(executionBuilder, executionCallback); 2495e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return false; 2505e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2515e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross sp<ExecutionCallback> fallbackCallback; 2525e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (executor->startComputeOnCpu(&fallbackCallback) != ANEURALNETWORKS_NO_ERROR) { 2535e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross cpuFallbackFull(executionBuilder, executionCallback); 2545e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return false; 2555e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2565e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross fallbackCallback->wait(); 2575e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (fallbackCallback->getStatus() != ErrorStatus::NONE) { 2585e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross cpuFallbackFull(executionBuilder, executionCallback); 2595e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return false; 2605e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2615e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return true; 2625e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross} 2635e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross 2645e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Grossstatic void asyncStartComputePartitioned(const ExecutionBuilder* executionBuilder, 2655e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const ExecutionPlan* plan, 266e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross std::shared_ptr<ExecutionPlan::Controller> controller, 2675e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross bool allowFallback, 2685e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const sp<ExecutionCallback>& executionCallback) { 269820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "ExecutionBuilder::startCompute (from plan, iteratively)"; 270e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross while (true) { 271e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross std::shared_ptr<StepExecutor> executor; 272820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "looking for next StepExecutor"; 273e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross int n = plan->next(controller, &executor); 2745e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (n != ANEURALNETWORKS_NO_ERROR) { 2755e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (allowFallback) { 2765e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross cpuFallbackFull(executionBuilder, executionCallback); 2775e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 2785e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->notify(ErrorStatus::GENERAL_FAILURE); 2795e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2805e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 2815e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2825e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (executor == nullptr) { 2835e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->notify(ErrorStatus::NONE); 284e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross return; 285e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross } 286e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross 287e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross sp<ExecutionCallback> stepCallback; 288e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross n = executor->startCompute(&stepCallback); 289e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross if (n != ANEURALNETWORKS_NO_ERROR) { 2905e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (allowFallback) { 2915e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (cpuFallbackPartial(executionBuilder, plan, controller, executionCallback)) { 2925e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Successfully executed one step on CPU. 2935e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross continue; 2945e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 2955e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Either successfully executed entire plan on 2965e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // CPU, or tried and failed to do so. 2975e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 2985e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2995e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 3005e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->notify(ErrorStatus::GENERAL_FAILURE); 3015e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 3025e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 303e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross } 304e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross stepCallback->wait(); 305e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross ErrorStatus status = stepCallback->getStatus(); 306e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross if (status != ErrorStatus::NONE) { 3075e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (allowFallback) { 3085e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (cpuFallbackPartial(executionBuilder, plan, controller, executionCallback)) { 3095e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Successfully executed one step on CPU. 3105e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross continue; 3115e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 3125e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Either successfully executed entire plan on 3135e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // CPU, or tried and failed to do so. 3145e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 3155e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 3165e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 3175e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->notify(status); 3185e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 3195e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 320e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross } 321e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross } 322e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross} 323e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross 324033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint ExecutionBuilder::startCompute(sp<ExecutionCallback>* synchronizationCallback) { 325033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = nullptr; 326425b2594c76e934dfdbc93209253e3c189571149David Gross 327707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // TODO validate that we have full types for all inputs and outputs, 328707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // that the graph is not cyclic, 329f1817c663af4f22bc089ef82cd50df4186422c42Yang Ni 3308b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& p : mInputs) { 331389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (p.state == ModelArgumentInfo::UNSPECIFIED) { 3323ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_startCompute not all inputs specified"; 333707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 334707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 335707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 3368b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& p : mOutputs) { 337389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (p.state == ModelArgumentInfo::UNSPECIFIED) { 3383ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_startCompute not all outputs specified"; 339707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 340707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 341707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 342707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 3435e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross#ifndef DISABLE_PARTITIONED_EXECUTION 3445e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross { 3455e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // TODO: Remove the non-plan-based path once we've fully integrated ExecutionPlan 3465e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // with the compilation and execution phases of the NN API? Or retain that path 3475e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // as a fallback in the case of partitioning failure? 3485e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // 3495e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // TODO: Entire plan-based-path should run in an asynchronous thread -- 3505e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // take the asynchronous thread logic out of startComputeOnCpu() and use 3515e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // it to wrap the plan-based-path. 352c2f1c1198c84f5a75fc2305935155f33b8ff5db2David Gross if (mPartitioning > 0) { 353c2f1c1198c84f5a75fc2305935155f33b8ff5db2David Gross const bool allowFallback = DeviceManager::partitioningAllowsFallback(mPartitioning); 3545e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross std::shared_ptr<ExecutionPlan::Controller> controller = mPlan->makeController(this); 3555e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (controller == nullptr) { 3565e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (!allowFallback) { 3575e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return ANEURALNETWORKS_OP_FAILED; 3585e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 3595e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 3605e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // TODO: use a thread pool 361b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 3625e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Prepare the callback for asynchronous execution. 3635e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // sp<ExecutionCallback> object is returned when the 3645e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // execution has been successfully launched, otherwise a 3655e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // nullptr is returned. The executionCallback is 3665e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // abstracted in the NN API as an "event". 3675e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross sp<ExecutionCallback> executionCallback = new ExecutionCallback(); 3685e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross std::thread thread(asyncStartComputePartitioned, this, mPlan, controller, 3695e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross allowFallback, 3705e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback); 3715e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->bind_thread(std::move(thread)); 3725e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross *synchronizationCallback = executionCallback; 3735e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return ANEURALNETWORKS_NO_ERROR; 3745e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 3751f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross } 3761f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross } 3775e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross#else 3785e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross { 3795e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Find a driver that can handle all the operations. 3805e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // TODO: Does not handle CPU fallback (which is tricky because 3815e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // StepExecutor::startCompute() is designed as 3825e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // asynchronous). 3835e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // TODO: Does not actually behave asynchronously (because 3845e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // StepExecutor::startCompute() isn't actually asynchronous 3855e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // on a device as opposed to a CPU). 3865e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross Model hidlModel; 3875e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross mModel->setHidlModel(&hidlModel); 3885e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const std::vector<std::shared_ptr<Device>>& devices = DeviceManager::get()->getDrivers(); 3895e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross for (const auto& device : devices) { 3905e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross hidl_vec<bool> supports; 391820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "Checking " << device->getName(); 3925e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross device->getSupportedOperations(hidlModel, &supports); 3935e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (std::find(supports.begin(), supports.end(), false) == supports.end()) { 394820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "ExecutionBuilder::startCompute (without plan) on " << device->getName(); 3955e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross StepExecutor executor(this, mModel, device->getInterface(), 3965e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross nullptr /* no IPreparedModel, so compile */); 3975e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executor.mapInputsAndOutputsTrivially(); 3985e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return executor.startCompute(synchronizationCallback); 3995e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 400ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet } 401ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet } 4025e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross#endif // DISABLE_PARTITIONED_EXECUTION 4035e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross 4045e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Run on the CPU. 405820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "ExecutionBuilder::startCompute (without plan) on CPU"; 406b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross StepExecutor executor(this, mModel, 40775886e77f9ca074173a49283b5c0a8c182d98977Michael Butler nullptr /* no VersionedIDevice, so CPU */, 408b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross nullptr /* no IPreparedModel */); 409b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross executor.mapInputsAndOutputsTrivially(); 410033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return executor.startCompute(synchronizationCallback); 411707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 412707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 413707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet// Figures out how to place each of the input or outputs in a buffer. This just does the layout, 4148b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet// it does not copy data. Aligns each input a bit. 415b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Grossint StepExecutor::allocatePointerArgumentsToPool(std::vector<ModelArgumentInfo>* args, 416b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross Memory* memory) { 4178b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t nextPoolIndex = mMemories.size(); 418707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet int64_t total = 0; 4198b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& info : *args) { 4208b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (info.state == ModelArgumentInfo::POINTER) { 42162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet DataLocation& loc = info.locationAndLength; 4228b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // TODO Good enough alignment? 4238b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet total += alignBytesNeeded(static_cast<uint32_t>(total), loc.length); 4248b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet loc.poolIndex = nextPoolIndex; 4258b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet loc.offset = static_cast<uint32_t>(total); 4268b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet total += loc.length; 4278b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 4288b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet }; 429707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (total > 0xFFFFFFFF) { 4303ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_startCompute Size of all inputs or outputs exceeds " 431707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet "2^32."; 432707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 433707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 4348b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet hidl_memory hidlMemory; 4358b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (total > 0) { 436389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet memory->create(total); // TODO check error 4378b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet mMemories.add(memory); 4388b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 439707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 440707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 441707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 44262cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouilletstatic void setRequestArgumentArray(const std::vector<ModelArgumentInfo>& argumentInfos, 443a4e2ee81d015b5e26897d18b18dd50044faba62fJean-Luc Brouillet hidl_vec<RequestArgument>* ioInfos) { 4448b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet size_t count = argumentInfos.size(); 4458b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet ioInfos->resize(count); 4468b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (size_t i = 0; i < count; i++) { 44762cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet const auto& info = argumentInfos[i]; 44862cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet (*ioInfos)[i] = { .hasNoValue = info.state == ModelArgumentInfo::HAS_NO_VALUE, 44962cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet .location = info.locationAndLength, 45062cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet .dimensions = info.dimensions, 45162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet }; 4528b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 4538b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet} 4548b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet 455b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David GrossStepExecutor::StepExecutor(const ExecutionBuilder* executionBuilder, 456b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross const ModelBuilder* model, 45775886e77f9ca074173a49283b5c0a8c182d98977Michael Butler VersionedIDevice* driver, sp<IPreparedModel> preparedModel) : 458b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mExecutionBuilder(executionBuilder), mModel(model), 459891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross mDriver(driver), mPreparedModel(preparedModel), 460891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross mInputs(model->inputCount()), mOutputs(model->outputCount()) {} 461b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 462b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Grossvoid StepExecutor::mapInputsAndOutputsTrivially() { 463b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mInputs = mExecutionBuilder->mInputs; 464b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mOutputs = mExecutionBuilder->mOutputs; 465b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mMemories = mExecutionBuilder->mMemories; 466b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross} 467b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 468891b10f7048c62a37a74c4b570be220089dfd55eDavid Grossvoid StepExecutor::mapInputOrOutput(const ModelArgumentInfo& builderInputOrOutput, 469891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross ModelArgumentInfo* executorInputOrOutput) { 470891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross *executorInputOrOutput = builderInputOrOutput; 471891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross switch (executorInputOrOutput->state) { 472891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross default: 473891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross nnAssert(!"unexpected ModelArgumentInfo::state"); 474891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross case ModelArgumentInfo::POINTER: 475891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross case ModelArgumentInfo::UNSPECIFIED: 476891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross break; 477891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross case ModelArgumentInfo::MEMORY: { 478891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const uint32_t builderPoolIndex = 47962cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet builderInputOrOutput.locationAndLength.poolIndex; 480891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const Memory* memory = mExecutionBuilder->mMemories[builderPoolIndex]; 481891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const uint32_t executorPoolIndex = mMemories.add(memory); 48262cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet executorInputOrOutput->locationAndLength.poolIndex = 483891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross executorPoolIndex; 484891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross break; 485891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 486891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 487891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross} 488891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross 48996811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Grossint StepExecutor::setInputOrOutputFromTemporaryMemory(const Operand& inputOrOutputOperand, 4908fb14e90ceb360adfbac0f708d27161b7c5b7fc5David Gross const Memory* memory, uint32_t offset, 49196811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross ModelArgumentInfo* inputOrOutputInfo) { 49296811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // Should be similar to 49396811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // ExecutionBuilder::setInputFromMemory() 49496811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // ExecutionBuilder::setOutputFromMemory() 49596811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 49696811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross uint32_t poolIndex = mMemories.add(memory); 4978fb14e90ceb360adfbac0f708d27161b7c5b7fc5David Gross return inputOrOutputInfo->setFromTemporaryMemory(inputOrOutputOperand, poolIndex, offset); 49896811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross} 49996811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 500c4c264098a728268ad28084ea6e0263d9c1d7868David Grossstatic void logArguments(const char* kind, const std::vector<ModelArgumentInfo> &args) { 501c4c264098a728268ad28084ea6e0263d9c1d7868David Gross for (unsigned i = 0; i < args.size(); i++) { 502c4c264098a728268ad28084ea6e0263d9c1d7868David Gross const auto& arg = args[i]; 503c4c264098a728268ad28084ea6e0263d9c1d7868David Gross std::string prefix = kind + std::string("[") + std::to_string(i) + "] = "; 504c4c264098a728268ad28084ea6e0263d9c1d7868David Gross switch (arg.state) { 505c4c264098a728268ad28084ea6e0263d9c1d7868David Gross case ModelArgumentInfo::POINTER: 506c4c264098a728268ad28084ea6e0263d9c1d7868David Gross VLOG(EXECUTION) << prefix << "POINTER(" << arg.buffer << ")"; 507c4c264098a728268ad28084ea6e0263d9c1d7868David Gross break; 508c4c264098a728268ad28084ea6e0263d9c1d7868David Gross case ModelArgumentInfo::MEMORY: 509c4c264098a728268ad28084ea6e0263d9c1d7868David Gross VLOG(EXECUTION) << prefix << "MEMORY(" 510c4c264098a728268ad28084ea6e0263d9c1d7868David Gross << "pool=" << arg.locationAndLength.poolIndex 511c4c264098a728268ad28084ea6e0263d9c1d7868David Gross << ", " 512c4c264098a728268ad28084ea6e0263d9c1d7868David Gross << "off=" << arg.locationAndLength.offset 513c4c264098a728268ad28084ea6e0263d9c1d7868David Gross << ")"; 514c4c264098a728268ad28084ea6e0263d9c1d7868David Gross break; 515c4c264098a728268ad28084ea6e0263d9c1d7868David Gross case ModelArgumentInfo::HAS_NO_VALUE: 516c4c264098a728268ad28084ea6e0263d9c1d7868David Gross VLOG(EXECUTION) << prefix << "HAS_NO_VALUE"; 517c4c264098a728268ad28084ea6e0263d9c1d7868David Gross break; 518c4c264098a728268ad28084ea6e0263d9c1d7868David Gross case ModelArgumentInfo::UNSPECIFIED: 519c4c264098a728268ad28084ea6e0263d9c1d7868David Gross VLOG(EXECUTION) << prefix << "UNSPECIFIED"; 520c4c264098a728268ad28084ea6e0263d9c1d7868David Gross break; 521c4c264098a728268ad28084ea6e0263d9c1d7868David Gross default: 522c4c264098a728268ad28084ea6e0263d9c1d7868David Gross VLOG(EXECUTION) << prefix << "state(" << arg.state << ")"; 523c4c264098a728268ad28084ea6e0263d9c1d7868David Gross break; 524c4c264098a728268ad28084ea6e0263d9c1d7868David Gross } 525c4c264098a728268ad28084ea6e0263d9c1d7868David Gross } 526c4c264098a728268ad28084ea6e0263d9c1d7868David Gross} 527c4c264098a728268ad28084ea6e0263d9c1d7868David Gross 528033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint StepExecutor::startCompute(sp<ExecutionCallback>* synchronizationCallback) { 529c4c264098a728268ad28084ea6e0263d9c1d7868David Gross if (VLOG_IS_ON(EXECUTION)) { 530c4c264098a728268ad28084ea6e0263d9c1d7868David Gross logArguments("input", mInputs); 531c4c264098a728268ad28084ea6e0263d9c1d7868David Gross logArguments("output", mOutputs); 532c4c264098a728268ad28084ea6e0263d9c1d7868David Gross } 533b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross if (mDriver == nullptr) { 534033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return startComputeOnCpu(synchronizationCallback); 535b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross } else { 536033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return startComputeOnDevice(synchronizationCallback); 537b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross } 538b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross} 539b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 540033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint StepExecutor::startComputeOnDevice(sp<ExecutionCallback>* synchronizationCallback) { 541b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross nnAssert(mDriver != nullptr); 542b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 543033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = nullptr; 544425b2594c76e934dfdbc93209253e3c189571149David Gross 545b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross // TODO: Remove the mPreparedModel == nullptr case once we've fully integrated 5461f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // ExecutionPlan with the compilation and execution phases of the NN API 547b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross if (mPreparedModel == nullptr) { 5481f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross Model model; 5491f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mModel->setHidlModel(&model); 5501f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross 5511f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // TODO Dangerous! In async, the model will outlive it here. Safe for now 552033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback(); 55375886e77f9ca074173a49283b5c0a8c182d98977Michael Butler ErrorStatus prepareLaunchStatus = mDriver->prepareModel(model, preparedModelCallback); 55475886e77f9ca074173a49283b5c0a8c182d98977Michael Butler if (prepareLaunchStatus != ErrorStatus::NONE) { 555033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return ANEURALNETWORKS_OP_FAILED; 556033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler } 5575f916fc9a7ae95f172492bdfe5344c37beff3a6fMichael Butler 558033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // Immediately synchronize with callback object for now 5591f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // TODO: change to asynchronous later 560033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler preparedModelCallback->wait(); 561033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); 562033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler mPreparedModel = preparedModelCallback->getPreparedModel(); 563033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler if (prepareReturnStatus != ErrorStatus::NONE || mPreparedModel == nullptr) { 5641f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross return ANEURALNETWORKS_OP_FAILED; 5651f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross } 566707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 567707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 568e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // We separate the input & output pools so that we reduce the copying done if we 569e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // do an eventual remoting (hidl_memory->update()). We could also use it to set 570e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // protection on read only memory but that's not currently done. 571e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross Memory inputPointerArguments; 572e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross Memory outputPointerArguments; 573e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross 5748b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // Layout the input and output data 575e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross int n = allocatePointerArgumentsToPool(&mInputs, &inputPointerArguments); 576707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 577707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return n; 578707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 579e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross n = allocatePointerArgumentsToPool(&mOutputs, &outputPointerArguments); 580707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 581707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return n; 582707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 583707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 5848b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // Copy the input data that was specified via a pointer. 585e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // inputPointerArguments.update(); 5868b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& info : mInputs) { 5878b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (info.state == ModelArgumentInfo::POINTER) { 58862cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet DataLocation& loc = info.locationAndLength; 5892150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet uint8_t* data = nullptr; 590e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross int n = inputPointerArguments.getPointer(&data); 5912150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 5922150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet return n; 5932150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet } 5948b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet memcpy(data + loc.offset, info.buffer, loc.length); 5958b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 596707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 597e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // TODO: Add inputPointerArguments.commit() and .update() at all the right places 598707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 599707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet Request request; 60062cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mInputs, &request.inputs); 60162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mOutputs, &request.outputs); 6028b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = mMemories.size(); 6038b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet request.pools.resize(count); 6048b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (uint32_t i = 0; i < count; i++) { 6058b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet request.pools[i] = mMemories[i]->getHidlMemory(); 6068b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 607707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 608033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // Prepare the callback for asynchronous execution. sp<ExecutionCallback> 609033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // object is returned when the execution has been successfully launched, 610033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // otherwise a nullptr is returned. The executionCallback is abstracted in 611033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // the NN API as an "event". 612033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // 613033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // The sp is used for ref-counting purposes. Without it, the HIDL service 614033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // could attempt to communicate with a dead callback object. 615689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // 616033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // TODO: Explain the "dead callback" problem further, either here or 617689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // in the design document. 618033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler sp<ExecutionCallback> executionCallback = new ExecutionCallback(); 619689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 620820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "Before mPreparedModel->execute() " << toString(request); 6213ced3cfd5b8f22b632c35f24e585c4847383b195David Gross // Execute. 622033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // TODO: What happens to the Callback if the service dies abnormally 623033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // -- won't that keep the Callback live forever, because the service 624689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // never has the opportunity to bump the reference count down? Or 625689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // maybe the HIDL infrastructure handles this magically? At worst, 626033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // it seems like this is a small memory leak, if the Callback stays 627689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // alive forever. 62806b5a0a85abe10b618a3a5ed023ac5c74678647eDavid Gross Return<ErrorStatus> executeStatus = mPreparedModel->execute(request, executionCallback); 62906b5a0a85abe10b618a3a5ed023ac5c74678647eDavid Gross if (!executeStatus.isOk() || executeStatus != ErrorStatus::NONE) { 630820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "**Execute failed**"; 631707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_OP_FAILED; 632707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 633707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 634689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: Remove this synchronization point when the block of code below is 635689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // removed. 636033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler executionCallback->wait(); 63706b5a0a85abe10b618a3a5ed023ac5c74678647eDavid Gross Return<ErrorStatus> callbackStatus = executionCallback->getStatus(); 63806b5a0a85abe10b618a3a5ed023ac5c74678647eDavid Gross if (!callbackStatus.isOk() || callbackStatus != ErrorStatus::NONE) { 639820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "**Execute async failed**"; 640689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler return ANEURALNETWORKS_OP_FAILED; 641689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler } 642689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 643707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // Copy the output data from shared memory to the output buffers. 644689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: Move this block of code somewhere else. It should not be in the 645689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // startCompute function. 646689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: outputMemory->update(); outputMemory->commit() 6478b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& info : mOutputs) { 6488b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (info.state == ModelArgumentInfo::POINTER) { 64962cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet DataLocation& loc = info.locationAndLength; 6502150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet uint8_t* data = nullptr; 651e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross int n = outputPointerArguments.getPointer(&data); 6522150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 6532150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet return n; 6542150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet } 6558b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet memcpy(info.buffer, data + loc.offset, loc.length); 6568b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 657707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 658820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "StepExecutor::startComputeOnDevice completed"; 659707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 660033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = executionCallback; 661707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 662707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 663707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 664689d892203c06c66c7bb2e374462a8434e40b75fMichael Butlerstatic void asyncStartComputeOnCpu(const Model& model, const Request& request, 6651da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet const std::vector<RunTimePoolInfo>& modelPoolInfos, 6661da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet const std::vector<RunTimePoolInfo>& requestPoolInfos, 667033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler const sp<IExecutionCallback>& executionCallback) { 668689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler CpuExecutor executor; 6691da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet int err = executor.run(model, request, modelPoolInfos, requestPoolInfos); 6705f916fc9a7ae95f172492bdfe5344c37beff3a6fMichael Butler ErrorStatus status = err == ANEURALNETWORKS_NO_ERROR ? 6715f916fc9a7ae95f172492bdfe5344c37beff3a6fMichael Butler ErrorStatus::NONE : ErrorStatus::GENERAL_FAILURE; 672033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler executionCallback->notify(status); 673689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler} 674689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 675033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint StepExecutor::startComputeOnCpu(sp<ExecutionCallback>* synchronizationCallback) { 676707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // TODO: use a thread pool 677689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 6781f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross Model model; 6791f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mModel->setHidlModel(&model); 6801f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross 681033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // Prepare the callback for asynchronous execution. sp<ExecutionCallback> 682033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // object is returned when the execution has been successfully launched, 683033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // otherwise a nullptr is returned. The executionCallback is abstracted in 684033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // the NN API as an "event". 685033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler sp<ExecutionCallback> executionCallback = new ExecutionCallback(); 686033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = nullptr; 687707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 6881da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet std::vector<RunTimePoolInfo> modelPoolInfos; 6891da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet if (!setRunTimePoolInfosFromHidlMemories(&modelPoolInfos, model.pools)) { 6901da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet return ANEURALNETWORKS_UNMAPPABLE; 6911da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet } 6921da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet 6931da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet std::vector<RunTimePoolInfo> requestPoolInfos; 6944cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross requestPoolInfos.reserve(mMemories.size()); 6954cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross bool fail = false; 6964cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross for (const Memory* mem : mMemories) { 6974cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross requestPoolInfos.emplace_back(mem->getHidlMemory(), &fail); 6984cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross } 6994cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross if (fail) { 7004cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross return ANEURALNETWORKS_UNMAPPABLE; 7018b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 7028b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // Create as many pools as there are input / output. 7031da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet auto fixPointerArguments = [&requestPoolInfos](std::vector<ModelArgumentInfo>& argumentInfos) { 7048b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (ModelArgumentInfo& argumentInfo : argumentInfos) { 7058b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (argumentInfo.state == ModelArgumentInfo::POINTER) { 70662cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet argumentInfo.locationAndLength.poolIndex = 7071da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet static_cast<uint32_t>(requestPoolInfos.size()); 70862cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet argumentInfo.locationAndLength.offset = 0; 7094cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross requestPoolInfos.emplace_back(static_cast<uint8_t*>(argumentInfo.buffer)); 7108b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 7118b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 7128b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet }; 7138b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet fixPointerArguments(mInputs); 7148b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet fixPointerArguments(mOutputs); 715707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 7168b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet Request request; 71762cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mInputs, &request.inputs); 71862cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mOutputs, &request.outputs); 719707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 720689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: should model be moved with a std::cref? 721689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler std::thread thread(asyncStartComputeOnCpu, model, std::move(request), 7221da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet std::move(modelPoolInfos), std::move(requestPoolInfos), 7231da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet executionCallback); 724033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler executionCallback->bind_thread(std::move(thread)); 725689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 726033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = executionCallback; 727689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler return ANEURALNETWORKS_NO_ERROR; 728707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 729707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 730389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} // namespace nn 731389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} // namespace android 732