ExecutionBuilder.cpp revision 1e9666208595bc251a8958155b1e41eca90b69db
1707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet/* 2707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * Copyright (C) 2017 The Android Open Source Project 3707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * 4707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * Licensed under the Apache License, Version 2.0 (the "License"); 5707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * you may not use this file except in compliance with the License. 6707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * You may obtain a copy of the License at 7707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * 8707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * http://www.apache.org/licenses/LICENSE-2.0 9707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * 10707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * Unless required by applicable law or agreed to in writing, software 11707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * distributed under the License is distributed on an "AS IS" BASIS, 12707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * See the License for the specific language governing permissions and 14707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * limitations under the License. 15707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet */ 16707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 173ced3cfd5b8f22b632c35f24e585c4847383b195David Gross#define LOG_TAG "ExecutionBuilder" 18707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 193ced3cfd5b8f22b632c35f24e585c4847383b195David Gross#include "ExecutionBuilder.h" 20707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 2183e24dc4706a5b7089881a55daf05b3924fab3b7David Gross#include "CompilationBuilder.h" 22707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "CpuExecutor.h" 23707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "HalInterfaces.h" 24707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "Manager.h" 25707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "ModelBuilder.h" 2607ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross#include "Utils.h" 27707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 28689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler#include <mutex> 29389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet#include <thread> 30689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler#include <vector> 31689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 32707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouilletnamespace android { 33707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouilletnamespace nn { 34707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 35389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouilletint ModelArgumentInfo::setFromPointer(const Operand& operand, 36389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet const ANeuralNetworksOperandType* type, void* data, 37389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t length) { 3843d018d1a66872031568857fa369024c9640a085Miao Wang if ((data == nullptr) != (length == 0)) { 3943d018d1a66872031568857fa369024c9640a085Miao Wang LOG(ERROR) << "Data pointer must be nullptr if and only if length is zero (data = " 4043d018d1a66872031568857fa369024c9640a085Miao Wang << data << ", length = " << length << ")"; 4143d018d1a66872031568857fa369024c9640a085Miao Wang return ANEURALNETWORKS_BAD_DATA; 42389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 4362cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet if (data == nullptr) { 4462cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet state = ModelArgumentInfo::HAS_NO_VALUE; 4562cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet } else { 4643d018d1a66872031568857fa369024c9640a085Miao Wang int n = updateDimensionInfo(operand, type); 4743d018d1a66872031568857fa369024c9640a085Miao Wang if (n != ANEURALNETWORKS_NO_ERROR) { 4843d018d1a66872031568857fa369024c9640a085Miao Wang return n; 4943d018d1a66872031568857fa369024c9640a085Miao Wang } 5043d018d1a66872031568857fa369024c9640a085Miao Wang uint32_t neededLength = sizeOfData(operand.type, dimensions); 5143d018d1a66872031568857fa369024c9640a085Miao Wang if (operand.type != OperandType::OEM && neededLength != length) { 5243d018d1a66872031568857fa369024c9640a085Miao Wang LOG(ERROR) << "Setting argument with invalid length: " << length 5343d018d1a66872031568857fa369024c9640a085Miao Wang << ", expected length: " << neededLength; 5443d018d1a66872031568857fa369024c9640a085Miao Wang return ANEURALNETWORKS_BAD_DATA; 5543d018d1a66872031568857fa369024c9640a085Miao Wang } 5662cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet state = ModelArgumentInfo::POINTER; 5762cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet } 58389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet buffer = data; 5962cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet locationAndLength = {.poolIndex = 0, .offset = 0, .length = length}; 60389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 61389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} 62389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet 63389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouilletint ModelArgumentInfo::setFromMemory(const Operand& operand, const ANeuralNetworksOperandType* type, 64389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t poolIndex, uint32_t offset, uint32_t length) { 65389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet int n = updateDimensionInfo(operand, type); 66389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 67389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return n; 68389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 6943d018d1a66872031568857fa369024c9640a085Miao Wang uint32_t neededLength = sizeOfData(operand.type, dimensions); 7043d018d1a66872031568857fa369024c9640a085Miao Wang if (operand.type != OperandType::OEM && neededLength != length) { 7143d018d1a66872031568857fa369024c9640a085Miao Wang LOG(ERROR) << "Setting argument with invalid length: " << length 7243d018d1a66872031568857fa369024c9640a085Miao Wang << ", expected length: " << neededLength; 7343d018d1a66872031568857fa369024c9640a085Miao Wang return ANEURALNETWORKS_BAD_DATA; 7443d018d1a66872031568857fa369024c9640a085Miao Wang } 7543d018d1a66872031568857fa369024c9640a085Miao Wang 76389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet state = ModelArgumentInfo::MEMORY; 7762cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet locationAndLength = {.poolIndex = poolIndex, .offset = offset, .length = length}; 78389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet buffer = nullptr; 79389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 80389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} 81389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet 828fb14e90ceb360adfbac0f708d27161b7c5b7fc5David Grossint ModelArgumentInfo::setFromTemporaryMemory(const Operand& operand, 838fb14e90ceb360adfbac0f708d27161b7c5b7fc5David Gross uint32_t poolIndex, uint32_t offset) { 8443d018d1a66872031568857fa369024c9640a085Miao Wang int n = updateDimensionInfo(operand, nullptr); 8543d018d1a66872031568857fa369024c9640a085Miao Wang if (n != ANEURALNETWORKS_NO_ERROR) { 8643d018d1a66872031568857fa369024c9640a085Miao Wang return n; 8743d018d1a66872031568857fa369024c9640a085Miao Wang } 8896811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross state = ModelArgumentInfo::MEMORY; 894d83c52f52613585f7b86368be762b2857f7460fDavid Gross locationAndLength = 908fb14e90ceb360adfbac0f708d27161b7c5b7fc5David Gross {.poolIndex = poolIndex, .offset = offset, .length = sizeOfData(operand)}; 9196811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross buffer = nullptr; 9296811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross return ANEURALNETWORKS_NO_ERROR; 9396811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross} 9496811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 95389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouilletint ModelArgumentInfo::updateDimensionInfo(const Operand& operand, 96389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet const ANeuralNetworksOperandType* newType) { 9743d018d1a66872031568857fa369024c9640a085Miao Wang nnAssert(dimensions.empty()); 98389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (newType == nullptr) { 9943d018d1a66872031568857fa369024c9640a085Miao Wang for (auto i : operand.dimensions) { 10043d018d1a66872031568857fa369024c9640a085Miao Wang if (i == 0) { 10143d018d1a66872031568857fa369024c9640a085Miao Wang LOG(ERROR) << "Setting input/output with unspecified dimensions"; 10243d018d1a66872031568857fa369024c9640a085Miao Wang return ANEURALNETWORKS_BAD_DATA; 10343d018d1a66872031568857fa369024c9640a085Miao Wang } 10443d018d1a66872031568857fa369024c9640a085Miao Wang } 10543d018d1a66872031568857fa369024c9640a085Miao Wang dimensions = operand.dimensions; 106389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } else { 107d2d0c031c43e8e5aafc75e8a652d79bcc2aaca99Jean-Luc Brouillet uint32_t count = newType->dimensionCount; 108389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (static_cast<OperandType>(newType->type) != operand.type || 109389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet count != operand.dimensions.size()) { 11043d018d1a66872031568857fa369024c9640a085Miao Wang LOG(ERROR) << "Setting input/output with incompatible types"; 111389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 112389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 11346a54e8a58e39c2fcb3448de04be76d95e744618Chenjie Luo 11446a54e8a58e39c2fcb3448de04be76d95e744618Chenjie Luo dimensions = hidl_vec<uint32_t>(count); 115389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet for (uint32_t i = 0; i < count; i++) { 11643d018d1a66872031568857fa369024c9640a085Miao Wang if (operand.dimensions[i] != 0 && operand.dimensions[i] != newType->dimensions[i]) { 11743d018d1a66872031568857fa369024c9640a085Miao Wang LOG(ERROR) << "Overriding a fully specified dimension is disallowed"; 11843d018d1a66872031568857fa369024c9640a085Miao Wang return ANEURALNETWORKS_BAD_DATA; 11943d018d1a66872031568857fa369024c9640a085Miao Wang } else { 12043d018d1a66872031568857fa369024c9640a085Miao Wang dimensions[i] = newType->dimensions[i]; 12143d018d1a66872031568857fa369024c9640a085Miao Wang } 122389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 123389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 124389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 125389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} 126389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet 1273ced3cfd5b8f22b632c35f24e585c4847383b195David GrossExecutionBuilder::ExecutionBuilder(const CompilationBuilder* compilation) : 12883e24dc4706a5b7089881a55daf05b3924fab3b7David Gross mModel(compilation->mModel), 1291f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mPlan(&compilation->mPlan), 130c2f1c1198c84f5a75fc2305935155f33b8ff5db2David Gross mPartitioning(compilation->mPartitioning), 13183e24dc4706a5b7089881a55daf05b3924fab3b7David Gross mInputs(mModel->inputCount()), 1321da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet mOutputs(mModel->outputCount()) { 133820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "ExecutionBuilder::ExecutionBuilder"; 134707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 135707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1363ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setInput(uint32_t index, const ANeuralNetworksOperandType* type, 137e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const void* buffer, size_t length) { 138707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mInputs.size()); 139707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (index >= count) { 1403ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setInput bad index " << index << " " << count; 141707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 142707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 143e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (type != nullptr) { 144e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet int n = validateOperandType(*type, "ANeuralNetworksExecution_setInput", false); 145e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 146e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return n; 147e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 148e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 149e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (length > 0xFFFFFFFF) { 150e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet LOG(ERROR) << "ANeuralNetworksExecution_setInput input exceeds max length " << length; 151e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 152e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 153e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet uint32_t l = static_cast<uint32_t>(length); 154389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return mInputs[index].setFromPointer(mModel->getInputOperand(index), type, 155e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const_cast<void*>(buffer), l); 156707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 157707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1583ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setInputFromMemory(uint32_t index, const ANeuralNetworksOperandType* type, 159e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const Memory* memory, size_t offset, size_t length) { 16096811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // Should be similar to StepExecutor::setInputOrOutputFromTemporaryMemory() 16196811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 1628b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mInputs.size()); 1638b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (index >= count) { 1643ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setInputFromMemory bad index " << index << " " 1658b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet << count; 1668b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 1678b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 168105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang if (!memory->validateSize(offset, length)) { 169105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang return ANEURALNETWORKS_BAD_DATA; 170105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang } 171e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet // TODO validate the rest 172389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t poolIndex = mMemories.add(memory); 173389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return mInputs[index].setFromMemory(mModel->getInputOperand(index), type, poolIndex, offset, 174389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet length); 175707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 176707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1773ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setOutput(uint32_t index, const ANeuralNetworksOperandType* type, void* buffer, 178e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet size_t length) { 179707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mOutputs.size()); 180707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (index >= count) { 1813ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setOutput bad index " << index << " " << count; 182707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 183707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 184e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (type != nullptr) { 185e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet int n = validateOperandType(*type, "ANeuralNetworksExecution_setOutput", false); 186e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 187e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return n; 188e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 189e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 190e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (length > 0xFFFFFFFF) { 191e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet LOG(ERROR) << "ANeuralNetworksExecution_setOutput input exceeds max length " << length; 192e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 193e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 194e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet uint32_t l = static_cast<uint32_t>(length); 195e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return mOutputs[index].setFromPointer(mModel->getOutputOperand(index), type, buffer, l); 196707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 197707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1983ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setOutputFromMemory(uint32_t index, const ANeuralNetworksOperandType* type, 199e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const Memory* memory, size_t offset, size_t length) { 20096811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // Should be similar to StepExecutor::setInputOrOutputFromTemporaryMemory() 20196811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 2028b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mOutputs.size()); 2038b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (index >= count) { 2043ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setOutputFromMemory bad index " << index << " " 2058b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet << count; 2068b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 2078b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 208105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang if (!memory->validateSize(offset, length)) { 209105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang return ANEURALNETWORKS_BAD_DATA; 210105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang } 211e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet // TODO validate the rest 212389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t poolIndex = mMemories.add(memory); 213389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return mOutputs[index].setFromMemory(mModel->getOutputOperand(index), type, poolIndex, offset, 214389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet length); 215707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 216707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 2175e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// Attempt synchronous execution of full model on CPU. 2185e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// Ensure that executionCallback->notify() is called. 2195e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Grossstatic void cpuFallbackFull(const ExecutionBuilder* executionBuilder, 2205e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const sp<ExecutionCallback>& executionCallback) { 221820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "cpuFallbackFull"; 2225e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross StepExecutor executor(executionBuilder, executionBuilder->getModel(), 22375886e77f9ca074173a49283b5c0a8c182d98977Michael Butler nullptr /* no VersionedIDevice, so CPU */, 2245e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross nullptr /* no IPreparedModel */); 2255e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executor.mapInputsAndOutputsTrivially(); 2265e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross sp<ExecutionCallback> fallbackCallback; 22707ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross int n = executor.startCompute(&fallbackCallback); 22807ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross if (n != ANEURALNETWORKS_NO_ERROR) { 22907ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross executionCallback->notify(convertResultCodeToErrorStatus(n)); 2305e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 2315e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2325e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross fallbackCallback->wait(); 2335e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->notify(fallbackCallback->getStatus()); 2345e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross} 2355e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross 2365e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// Attempt synchronous execution on CPU. 2375e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// (1) First, attempt to execute this step on CPU. If successful, 2385e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// return true. (Do not call executionCallback->notify().) 2395e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// (2) If unsuccessful, attempt to execute the full model on CPU, 2405e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// ensure that executionCallback->notify() is called, and return 2415e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// false. 2425e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Grossstatic bool cpuFallbackPartial(const ExecutionBuilder* executionBuilder, 2435e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const ExecutionPlan* plan, 2445e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross std::shared_ptr<ExecutionPlan::Controller> controller, 2455e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const sp<ExecutionCallback>& executionCallback) { 246820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "cpuFallbackPartial"; 2475e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross std::shared_ptr<StepExecutor> executor; 2485e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross int n = plan->fallback(controller, &executor); 2495e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (n != ANEURALNETWORKS_NO_ERROR || executor->isCpu()) { 2505e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross cpuFallbackFull(executionBuilder, executionCallback); 2515e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return false; 2525e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2535e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross sp<ExecutionCallback> fallbackCallback; 2545e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (executor->startComputeOnCpu(&fallbackCallback) != ANEURALNETWORKS_NO_ERROR) { 2555e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross cpuFallbackFull(executionBuilder, executionCallback); 2565e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return false; 2575e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2585e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross fallbackCallback->wait(); 2595e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (fallbackCallback->getStatus() != ErrorStatus::NONE) { 2605e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross cpuFallbackFull(executionBuilder, executionCallback); 2615e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return false; 2625e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2635e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return true; 2645e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross} 2655e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross 2665e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Grossstatic void asyncStartComputePartitioned(const ExecutionBuilder* executionBuilder, 2675e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const ExecutionPlan* plan, 268e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross std::shared_ptr<ExecutionPlan::Controller> controller, 2695e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross bool allowFallback, 2705e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const sp<ExecutionCallback>& executionCallback) { 271820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "ExecutionBuilder::startCompute (from plan, iteratively)"; 272e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross while (true) { 273e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross std::shared_ptr<StepExecutor> executor; 274820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "looking for next StepExecutor"; 275e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross int n = plan->next(controller, &executor); 2765e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (n != ANEURALNETWORKS_NO_ERROR) { 2775e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (allowFallback) { 2785e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross cpuFallbackFull(executionBuilder, executionCallback); 2795e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 28007ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross executionCallback->notify(convertResultCodeToErrorStatus(n)); 2815e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2825e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 2835e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2845e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (executor == nullptr) { 2855e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->notify(ErrorStatus::NONE); 286e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross return; 287e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross } 288e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross 289e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross sp<ExecutionCallback> stepCallback; 290e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross n = executor->startCompute(&stepCallback); 291e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross if (n != ANEURALNETWORKS_NO_ERROR) { 2925e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (allowFallback) { 2935e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (cpuFallbackPartial(executionBuilder, plan, controller, executionCallback)) { 2945e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Successfully executed one step on CPU. 2955e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross continue; 2965e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 2975e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Either successfully executed entire plan on 2985e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // CPU, or tried and failed to do so. 2995e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 3005e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 3015e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 30207ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross executionCallback->notify(convertResultCodeToErrorStatus(n)); 3035e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 3045e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 305e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross } 306e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross stepCallback->wait(); 307e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross ErrorStatus status = stepCallback->getStatus(); 308e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross if (status != ErrorStatus::NONE) { 3095e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (allowFallback) { 3105e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (cpuFallbackPartial(executionBuilder, plan, controller, executionCallback)) { 3115e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Successfully executed one step on CPU. 3125e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross continue; 3135e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 3145e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Either successfully executed entire plan on 3155e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // CPU, or tried and failed to do so. 3165e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 3175e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 3185e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 3195e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->notify(status); 3205e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 3215e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 322e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross } 323e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross } 324e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross} 325e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross 326033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint ExecutionBuilder::startCompute(sp<ExecutionCallback>* synchronizationCallback) { 327033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = nullptr; 328425b2594c76e934dfdbc93209253e3c189571149David Gross 329707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // TODO validate that we have full types for all inputs and outputs, 330707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // that the graph is not cyclic, 331f1817c663af4f22bc089ef82cd50df4186422c42Yang Ni 3328b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& p : mInputs) { 333389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (p.state == ModelArgumentInfo::UNSPECIFIED) { 3343ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_startCompute not all inputs specified"; 335707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 336707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 337707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 3388b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& p : mOutputs) { 339389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (p.state == ModelArgumentInfo::UNSPECIFIED) { 3403ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_startCompute not all outputs specified"; 341707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 342707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 343707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 344707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 3455e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross#ifndef DISABLE_PARTITIONED_EXECUTION 3465e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross { 3475e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // TODO: Remove the non-plan-based path once we've fully integrated ExecutionPlan 3485e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // with the compilation and execution phases of the NN API? Or retain that path 3495e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // as a fallback in the case of partitioning failure? 3505e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // 3515e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // TODO: Entire plan-based-path should run in an asynchronous thread -- 3525e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // take the asynchronous thread logic out of startComputeOnCpu() and use 3535e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // it to wrap the plan-based-path. 354c2f1c1198c84f5a75fc2305935155f33b8ff5db2David Gross if (mPartitioning > 0) { 355c2f1c1198c84f5a75fc2305935155f33b8ff5db2David Gross const bool allowFallback = DeviceManager::partitioningAllowsFallback(mPartitioning); 3565e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross std::shared_ptr<ExecutionPlan::Controller> controller = mPlan->makeController(this); 3575e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (controller == nullptr) { 3585e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (!allowFallback) { 3595e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return ANEURALNETWORKS_OP_FAILED; 3605e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 3615e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 3625e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // TODO: use a thread pool 363b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 3645e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Prepare the callback for asynchronous execution. 3655e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // sp<ExecutionCallback> object is returned when the 3665e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // execution has been successfully launched, otherwise a 3675e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // nullptr is returned. The executionCallback is 3685e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // abstracted in the NN API as an "event". 3695e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross sp<ExecutionCallback> executionCallback = new ExecutionCallback(); 3705e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross std::thread thread(asyncStartComputePartitioned, this, mPlan, controller, 3715e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross allowFallback, 3725e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback); 3735e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->bind_thread(std::move(thread)); 3745e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross *synchronizationCallback = executionCallback; 3755e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return ANEURALNETWORKS_NO_ERROR; 3765e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 3771f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross } 3781f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross } 3795e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross#else 3805e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross { 3815e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Find a driver that can handle all the operations. 3825e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // TODO: Does not handle CPU fallback (which is tricky because 3835e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // StepExecutor::startCompute() is designed as 3845e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // asynchronous). 3855e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // TODO: Does not actually behave asynchronously (because 3865e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // StepExecutor::startCompute() isn't actually asynchronous 3875e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // on a device as opposed to a CPU). 3885e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross Model hidlModel; 3895e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross mModel->setHidlModel(&hidlModel); 3905e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const std::vector<std::shared_ptr<Device>>& devices = DeviceManager::get()->getDrivers(); 3915e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross for (const auto& device : devices) { 3925e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross hidl_vec<bool> supports; 393820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "Checking " << device->getName(); 3945e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross device->getSupportedOperations(hidlModel, &supports); 3955e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (std::find(supports.begin(), supports.end(), false) == supports.end()) { 396820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "ExecutionBuilder::startCompute (without plan) on " << device->getName(); 3975e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross StepExecutor executor(this, mModel, device->getInterface(), 3985e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross nullptr /* no IPreparedModel, so compile */); 3995e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executor.mapInputsAndOutputsTrivially(); 4005e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return executor.startCompute(synchronizationCallback); 4015e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 402ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet } 403ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet } 4045e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross#endif // DISABLE_PARTITIONED_EXECUTION 4055e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross 4065e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Run on the CPU. 407820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "ExecutionBuilder::startCompute (without plan) on CPU"; 408b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross StepExecutor executor(this, mModel, 40975886e77f9ca074173a49283b5c0a8c182d98977Michael Butler nullptr /* no VersionedIDevice, so CPU */, 410b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross nullptr /* no IPreparedModel */); 411b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross executor.mapInputsAndOutputsTrivially(); 412033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return executor.startCompute(synchronizationCallback); 413707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 414707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 415707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet// Figures out how to place each of the input or outputs in a buffer. This just does the layout, 4168b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet// it does not copy data. Aligns each input a bit. 417b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Grossint StepExecutor::allocatePointerArgumentsToPool(std::vector<ModelArgumentInfo>* args, 418b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross Memory* memory) { 4198b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t nextPoolIndex = mMemories.size(); 420707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet int64_t total = 0; 4218b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& info : *args) { 4228b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (info.state == ModelArgumentInfo::POINTER) { 42362cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet DataLocation& loc = info.locationAndLength; 4248b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // TODO Good enough alignment? 4258b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet total += alignBytesNeeded(static_cast<uint32_t>(total), loc.length); 4268b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet loc.poolIndex = nextPoolIndex; 4278b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet loc.offset = static_cast<uint32_t>(total); 4288b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet total += loc.length; 4298b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 4308b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet }; 431707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (total > 0xFFFFFFFF) { 4323ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_startCompute Size of all inputs or outputs exceeds " 433707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet "2^32."; 434707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 435707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 4368b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet hidl_memory hidlMemory; 4378b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (total > 0) { 438389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet memory->create(total); // TODO check error 4398b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet mMemories.add(memory); 4408b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 441707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 442707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 443707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 44462cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouilletstatic void setRequestArgumentArray(const std::vector<ModelArgumentInfo>& argumentInfos, 445a4e2ee81d015b5e26897d18b18dd50044faba62fJean-Luc Brouillet hidl_vec<RequestArgument>* ioInfos) { 4468b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet size_t count = argumentInfos.size(); 4478b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet ioInfos->resize(count); 4488b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (size_t i = 0; i < count; i++) { 44962cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet const auto& info = argumentInfos[i]; 45062cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet (*ioInfos)[i] = { .hasNoValue = info.state == ModelArgumentInfo::HAS_NO_VALUE, 45162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet .location = info.locationAndLength, 45262cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet .dimensions = info.dimensions, 45362cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet }; 4548b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 4558b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet} 4568b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet 457b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David GrossStepExecutor::StepExecutor(const ExecutionBuilder* executionBuilder, 458b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross const ModelBuilder* model, 45975886e77f9ca074173a49283b5c0a8c182d98977Michael Butler VersionedIDevice* driver, sp<IPreparedModel> preparedModel) : 460b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mExecutionBuilder(executionBuilder), mModel(model), 461891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross mDriver(driver), mPreparedModel(preparedModel), 462891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross mInputs(model->inputCount()), mOutputs(model->outputCount()) {} 463b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 464b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Grossvoid StepExecutor::mapInputsAndOutputsTrivially() { 465b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mInputs = mExecutionBuilder->mInputs; 466b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mOutputs = mExecutionBuilder->mOutputs; 467b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mMemories = mExecutionBuilder->mMemories; 468b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross} 469b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 470891b10f7048c62a37a74c4b570be220089dfd55eDavid Grossvoid StepExecutor::mapInputOrOutput(const ModelArgumentInfo& builderInputOrOutput, 471891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross ModelArgumentInfo* executorInputOrOutput) { 472891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross *executorInputOrOutput = builderInputOrOutput; 473891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross switch (executorInputOrOutput->state) { 474891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross default: 475891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross nnAssert(!"unexpected ModelArgumentInfo::state"); 476891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross case ModelArgumentInfo::POINTER: 477891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross case ModelArgumentInfo::UNSPECIFIED: 478891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross break; 479891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross case ModelArgumentInfo::MEMORY: { 480891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const uint32_t builderPoolIndex = 48162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet builderInputOrOutput.locationAndLength.poolIndex; 482891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const Memory* memory = mExecutionBuilder->mMemories[builderPoolIndex]; 483891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const uint32_t executorPoolIndex = mMemories.add(memory); 48462cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet executorInputOrOutput->locationAndLength.poolIndex = 485891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross executorPoolIndex; 486891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross break; 487891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 488891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 489891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross} 490891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross 49196811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Grossint StepExecutor::setInputOrOutputFromTemporaryMemory(const Operand& inputOrOutputOperand, 4928fb14e90ceb360adfbac0f708d27161b7c5b7fc5David Gross const Memory* memory, uint32_t offset, 49396811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross ModelArgumentInfo* inputOrOutputInfo) { 49496811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // Should be similar to 49596811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // ExecutionBuilder::setInputFromMemory() 49696811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // ExecutionBuilder::setOutputFromMemory() 49796811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 49896811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross uint32_t poolIndex = mMemories.add(memory); 4998fb14e90ceb360adfbac0f708d27161b7c5b7fc5David Gross return inputOrOutputInfo->setFromTemporaryMemory(inputOrOutputOperand, poolIndex, offset); 50096811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross} 50196811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 502c4c264098a728268ad28084ea6e0263d9c1d7868David Grossstatic void logArguments(const char* kind, const std::vector<ModelArgumentInfo> &args) { 503c4c264098a728268ad28084ea6e0263d9c1d7868David Gross for (unsigned i = 0; i < args.size(); i++) { 504c4c264098a728268ad28084ea6e0263d9c1d7868David Gross const auto& arg = args[i]; 505c4c264098a728268ad28084ea6e0263d9c1d7868David Gross std::string prefix = kind + std::string("[") + std::to_string(i) + "] = "; 506c4c264098a728268ad28084ea6e0263d9c1d7868David Gross switch (arg.state) { 507c4c264098a728268ad28084ea6e0263d9c1d7868David Gross case ModelArgumentInfo::POINTER: 508c4c264098a728268ad28084ea6e0263d9c1d7868David Gross VLOG(EXECUTION) << prefix << "POINTER(" << arg.buffer << ")"; 509c4c264098a728268ad28084ea6e0263d9c1d7868David Gross break; 510c4c264098a728268ad28084ea6e0263d9c1d7868David Gross case ModelArgumentInfo::MEMORY: 511c4c264098a728268ad28084ea6e0263d9c1d7868David Gross VLOG(EXECUTION) << prefix << "MEMORY(" 512c4c264098a728268ad28084ea6e0263d9c1d7868David Gross << "pool=" << arg.locationAndLength.poolIndex 513c4c264098a728268ad28084ea6e0263d9c1d7868David Gross << ", " 514c4c264098a728268ad28084ea6e0263d9c1d7868David Gross << "off=" << arg.locationAndLength.offset 515c4c264098a728268ad28084ea6e0263d9c1d7868David Gross << ")"; 516c4c264098a728268ad28084ea6e0263d9c1d7868David Gross break; 517c4c264098a728268ad28084ea6e0263d9c1d7868David Gross case ModelArgumentInfo::HAS_NO_VALUE: 518c4c264098a728268ad28084ea6e0263d9c1d7868David Gross VLOG(EXECUTION) << prefix << "HAS_NO_VALUE"; 519c4c264098a728268ad28084ea6e0263d9c1d7868David Gross break; 520c4c264098a728268ad28084ea6e0263d9c1d7868David Gross case ModelArgumentInfo::UNSPECIFIED: 521c4c264098a728268ad28084ea6e0263d9c1d7868David Gross VLOG(EXECUTION) << prefix << "UNSPECIFIED"; 522c4c264098a728268ad28084ea6e0263d9c1d7868David Gross break; 523c4c264098a728268ad28084ea6e0263d9c1d7868David Gross default: 524c4c264098a728268ad28084ea6e0263d9c1d7868David Gross VLOG(EXECUTION) << prefix << "state(" << arg.state << ")"; 525c4c264098a728268ad28084ea6e0263d9c1d7868David Gross break; 526c4c264098a728268ad28084ea6e0263d9c1d7868David Gross } 527c4c264098a728268ad28084ea6e0263d9c1d7868David Gross } 528c4c264098a728268ad28084ea6e0263d9c1d7868David Gross} 529c4c264098a728268ad28084ea6e0263d9c1d7868David Gross 530033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint StepExecutor::startCompute(sp<ExecutionCallback>* synchronizationCallback) { 531c4c264098a728268ad28084ea6e0263d9c1d7868David Gross if (VLOG_IS_ON(EXECUTION)) { 532c4c264098a728268ad28084ea6e0263d9c1d7868David Gross logArguments("input", mInputs); 533c4c264098a728268ad28084ea6e0263d9c1d7868David Gross logArguments("output", mOutputs); 534c4c264098a728268ad28084ea6e0263d9c1d7868David Gross } 535b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross if (mDriver == nullptr) { 536033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return startComputeOnCpu(synchronizationCallback); 537b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross } else { 538033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return startComputeOnDevice(synchronizationCallback); 539b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross } 540b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross} 541b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 542033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint StepExecutor::startComputeOnDevice(sp<ExecutionCallback>* synchronizationCallback) { 543b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross nnAssert(mDriver != nullptr); 544b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 545033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = nullptr; 546425b2594c76e934dfdbc93209253e3c189571149David Gross 547b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross // TODO: Remove the mPreparedModel == nullptr case once we've fully integrated 5481f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // ExecutionPlan with the compilation and execution phases of the NN API 549b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross if (mPreparedModel == nullptr) { 5501f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross Model model; 5511f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mModel->setHidlModel(&model); 5521f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross 5531f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // TODO Dangerous! In async, the model will outlive it here. Safe for now 554033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback(); 5551e9666208595bc251a8958155b1e41eca90b69dbMichael Butler // TODO(butlermichael): Propagate user preference to this point instead of 5561e9666208595bc251a8958155b1e41eca90b69dbMichael Butler // using default value of ANEURALNETWORKS_PREFER_FAST_SINGLE_ANSWER, or 5571e9666208595bc251a8958155b1e41eca90b69dbMichael Butler // remove this entire block of code since it is a stale path that is only 5581e9666208595bc251a8958155b1e41eca90b69dbMichael Butler // encountered on an #if-removed code. 5591e9666208595bc251a8958155b1e41eca90b69dbMichael Butler ExecutionPreference preference = 5601e9666208595bc251a8958155b1e41eca90b69dbMichael Butler static_cast<ExecutionPreference>(ANEURALNETWORKS_PREFER_FAST_SINGLE_ANSWER); 5611e9666208595bc251a8958155b1e41eca90b69dbMichael Butler ErrorStatus prepareLaunchStatus = mDriver->prepareModel(model, preference, 5621e9666208595bc251a8958155b1e41eca90b69dbMichael Butler preparedModelCallback); 56375886e77f9ca074173a49283b5c0a8c182d98977Michael Butler if (prepareLaunchStatus != ErrorStatus::NONE) { 56407ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross return convertErrorStatusToResultCode(prepareLaunchStatus); 565033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler } 5665f916fc9a7ae95f172492bdfe5344c37beff3a6fMichael Butler 567033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // Immediately synchronize with callback object for now 5681f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // TODO: change to asynchronous later 569033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler preparedModelCallback->wait(); 570033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); 571033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler mPreparedModel = preparedModelCallback->getPreparedModel(); 57207ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross if (prepareReturnStatus != ErrorStatus::NONE) { 57307ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross return convertErrorStatusToResultCode(prepareReturnStatus); 57407ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross } 57507ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross if (mPreparedModel == nullptr) { 5761f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross return ANEURALNETWORKS_OP_FAILED; 5771f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross } 578707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 579707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 580e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // We separate the input & output pools so that we reduce the copying done if we 581e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // do an eventual remoting (hidl_memory->update()). We could also use it to set 582e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // protection on read only memory but that's not currently done. 583e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross Memory inputPointerArguments; 584e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross Memory outputPointerArguments; 585e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross 5868b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // Layout the input and output data 587e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross int n = allocatePointerArgumentsToPool(&mInputs, &inputPointerArguments); 588707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 589707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return n; 590707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 591e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross n = allocatePointerArgumentsToPool(&mOutputs, &outputPointerArguments); 592707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 593707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return n; 594707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 595707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 5968b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // Copy the input data that was specified via a pointer. 597e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // inputPointerArguments.update(); 5988b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& info : mInputs) { 5998b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (info.state == ModelArgumentInfo::POINTER) { 60062cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet DataLocation& loc = info.locationAndLength; 6012150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet uint8_t* data = nullptr; 602e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross int n = inputPointerArguments.getPointer(&data); 6032150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 6042150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet return n; 6052150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet } 6068b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet memcpy(data + loc.offset, info.buffer, loc.length); 6078b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 608707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 609e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // TODO: Add inputPointerArguments.commit() and .update() at all the right places 610707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 611707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet Request request; 61262cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mInputs, &request.inputs); 61362cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mOutputs, &request.outputs); 6148b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = mMemories.size(); 6158b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet request.pools.resize(count); 6168b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (uint32_t i = 0; i < count; i++) { 6178b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet request.pools[i] = mMemories[i]->getHidlMemory(); 6188b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 619707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 620033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // Prepare the callback for asynchronous execution. sp<ExecutionCallback> 621033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // object is returned when the execution has been successfully launched, 622033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // otherwise a nullptr is returned. The executionCallback is abstracted in 623033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // the NN API as an "event". 624033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // 625033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // The sp is used for ref-counting purposes. Without it, the HIDL service 626033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // could attempt to communicate with a dead callback object. 627689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // 628033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // TODO: Explain the "dead callback" problem further, either here or 629689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // in the design document. 630033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler sp<ExecutionCallback> executionCallback = new ExecutionCallback(); 631689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 632820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "Before mPreparedModel->execute() " << toString(request); 6333ced3cfd5b8f22b632c35f24e585c4847383b195David Gross // Execute. 634033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // TODO: What happens to the Callback if the service dies abnormally 635033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // -- won't that keep the Callback live forever, because the service 636689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // never has the opportunity to bump the reference count down? Or 637689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // maybe the HIDL infrastructure handles this magically? At worst, 638033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // it seems like this is a small memory leak, if the Callback stays 639689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // alive forever. 64006b5a0a85abe10b618a3a5ed023ac5c74678647eDavid Gross Return<ErrorStatus> executeStatus = mPreparedModel->execute(request, executionCallback); 64106b5a0a85abe10b618a3a5ed023ac5c74678647eDavid Gross if (!executeStatus.isOk() || executeStatus != ErrorStatus::NONE) { 642820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "**Execute failed**"; 64307ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross return executeStatus.isOk() 64407ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross ? convertErrorStatusToResultCode(executeStatus) 64507ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross : ANEURALNETWORKS_OP_FAILED; 646707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 647707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 648689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: Remove this synchronization point when the block of code below is 649689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // removed. 650033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler executionCallback->wait(); 65106b5a0a85abe10b618a3a5ed023ac5c74678647eDavid Gross Return<ErrorStatus> callbackStatus = executionCallback->getStatus(); 65206b5a0a85abe10b618a3a5ed023ac5c74678647eDavid Gross if (!callbackStatus.isOk() || callbackStatus != ErrorStatus::NONE) { 653820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "**Execute async failed**"; 65407ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross return callbackStatus.isOk() 65507ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross ? convertErrorStatusToResultCode(callbackStatus) 65607ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross : ANEURALNETWORKS_OP_FAILED; 657689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler } 658689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 659707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // Copy the output data from shared memory to the output buffers. 660689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: Move this block of code somewhere else. It should not be in the 661689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // startCompute function. 662689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: outputMemory->update(); outputMemory->commit() 6638b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& info : mOutputs) { 6648b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (info.state == ModelArgumentInfo::POINTER) { 66562cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet DataLocation& loc = info.locationAndLength; 6662150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet uint8_t* data = nullptr; 667e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross int n = outputPointerArguments.getPointer(&data); 6682150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 6692150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet return n; 6702150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet } 6718b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet memcpy(info.buffer, data + loc.offset, loc.length); 6728b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 673707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 674820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "StepExecutor::startComputeOnDevice completed"; 675707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 676033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = executionCallback; 677707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 678707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 679707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 680689d892203c06c66c7bb2e374462a8434e40b75fMichael Butlerstatic void asyncStartComputeOnCpu(const Model& model, const Request& request, 6811da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet const std::vector<RunTimePoolInfo>& modelPoolInfos, 6821da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet const std::vector<RunTimePoolInfo>& requestPoolInfos, 683033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler const sp<IExecutionCallback>& executionCallback) { 684689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler CpuExecutor executor; 6851da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet int err = executor.run(model, request, modelPoolInfos, requestPoolInfos); 68607ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross executionCallback->notify(convertResultCodeToErrorStatus(err)); 687689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler} 688689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 689033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint StepExecutor::startComputeOnCpu(sp<ExecutionCallback>* synchronizationCallback) { 690707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // TODO: use a thread pool 691689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 6921f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross Model model; 6931f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mModel->setHidlModel(&model); 6941f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross 695033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // Prepare the callback for asynchronous execution. sp<ExecutionCallback> 696033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // object is returned when the execution has been successfully launched, 697033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // otherwise a nullptr is returned. The executionCallback is abstracted in 698033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // the NN API as an "event". 699033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler sp<ExecutionCallback> executionCallback = new ExecutionCallback(); 700033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = nullptr; 701707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 7021da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet std::vector<RunTimePoolInfo> modelPoolInfos; 7031da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet if (!setRunTimePoolInfosFromHidlMemories(&modelPoolInfos, model.pools)) { 7041da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet return ANEURALNETWORKS_UNMAPPABLE; 7051da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet } 7061da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet 7071da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet std::vector<RunTimePoolInfo> requestPoolInfos; 7084cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross requestPoolInfos.reserve(mMemories.size()); 7094cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross bool fail = false; 7104cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross for (const Memory* mem : mMemories) { 7114cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross requestPoolInfos.emplace_back(mem->getHidlMemory(), &fail); 7124cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross } 7134cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross if (fail) { 7144cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross return ANEURALNETWORKS_UNMAPPABLE; 7158b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 7168b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // Create as many pools as there are input / output. 7171da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet auto fixPointerArguments = [&requestPoolInfos](std::vector<ModelArgumentInfo>& argumentInfos) { 7188b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (ModelArgumentInfo& argumentInfo : argumentInfos) { 7198b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (argumentInfo.state == ModelArgumentInfo::POINTER) { 72062cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet argumentInfo.locationAndLength.poolIndex = 7211da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet static_cast<uint32_t>(requestPoolInfos.size()); 72262cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet argumentInfo.locationAndLength.offset = 0; 7234cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross requestPoolInfos.emplace_back(static_cast<uint8_t*>(argumentInfo.buffer)); 7248b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 7258b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 7268b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet }; 7278b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet fixPointerArguments(mInputs); 7288b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet fixPointerArguments(mOutputs); 729707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 7308b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet Request request; 73162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mInputs, &request.inputs); 73262cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mOutputs, &request.outputs); 733707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 734689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: should model be moved with a std::cref? 735689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler std::thread thread(asyncStartComputeOnCpu, model, std::move(request), 7361da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet std::move(modelPoolInfos), std::move(requestPoolInfos), 7371da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet executionCallback); 738033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler executionCallback->bind_thread(std::move(thread)); 739689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 740033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = executionCallback; 741689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler return ANEURALNETWORKS_NO_ERROR; 742707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 743707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 744389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} // namespace nn 745389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} // namespace android 746