1707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet/* 2707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * Copyright (C) 2017 The Android Open Source Project 3707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * 4707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * Licensed under the Apache License, Version 2.0 (the "License"); 5707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * you may not use this file except in compliance with the License. 6707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * You may obtain a copy of the License at 7707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * 8707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * http://www.apache.org/licenses/LICENSE-2.0 9707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * 10707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * Unless required by applicable law or agreed to in writing, software 11707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * distributed under the License is distributed on an "AS IS" BASIS, 12707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * See the License for the specific language governing permissions and 14707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * limitations under the License. 15707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet */ 16707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 173ced3cfd5b8f22b632c35f24e585c4847383b195David Gross#define LOG_TAG "ExecutionBuilder" 18707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 193ced3cfd5b8f22b632c35f24e585c4847383b195David Gross#include "ExecutionBuilder.h" 20707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 2183e24dc4706a5b7089881a55daf05b3924fab3b7David Gross#include "CompilationBuilder.h" 22707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "CpuExecutor.h" 23707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "HalInterfaces.h" 24707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "Manager.h" 25707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "ModelBuilder.h" 2607ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross#include "Utils.h" 27707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 28689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler#include <mutex> 29389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet#include <thread> 30689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler#include <vector> 31689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 32707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouilletnamespace android { 33707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouilletnamespace nn { 34707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 35389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouilletint ModelArgumentInfo::setFromPointer(const Operand& operand, 36389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet const ANeuralNetworksOperandType* type, void* data, 37389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t length) { 3843d018d1a66872031568857fa369024c9640a085Miao Wang if ((data == nullptr) != (length == 0)) { 3964031fa26110e6d57896bfdbe1cd58e69cde3dbbMiao Wang const char* dataPtrMsg = data ? "NOT_NULLPTR" : "NULLPTR"; 4043d018d1a66872031568857fa369024c9640a085Miao Wang LOG(ERROR) << "Data pointer must be nullptr if and only if length is zero (data = " 4164031fa26110e6d57896bfdbe1cd58e69cde3dbbMiao Wang << dataPtrMsg << ", length = " << length << ")"; 4243d018d1a66872031568857fa369024c9640a085Miao Wang return ANEURALNETWORKS_BAD_DATA; 43389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 4462cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet if (data == nullptr) { 4562cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet state = ModelArgumentInfo::HAS_NO_VALUE; 4662cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet } else { 4743d018d1a66872031568857fa369024c9640a085Miao Wang int n = updateDimensionInfo(operand, type); 4843d018d1a66872031568857fa369024c9640a085Miao Wang if (n != ANEURALNETWORKS_NO_ERROR) { 4943d018d1a66872031568857fa369024c9640a085Miao Wang return n; 5043d018d1a66872031568857fa369024c9640a085Miao Wang } 5143d018d1a66872031568857fa369024c9640a085Miao Wang uint32_t neededLength = sizeOfData(operand.type, dimensions); 5243d018d1a66872031568857fa369024c9640a085Miao Wang if (operand.type != OperandType::OEM && neededLength != length) { 5343d018d1a66872031568857fa369024c9640a085Miao Wang LOG(ERROR) << "Setting argument with invalid length: " << length 5443d018d1a66872031568857fa369024c9640a085Miao Wang << ", expected length: " << neededLength; 5543d018d1a66872031568857fa369024c9640a085Miao Wang return ANEURALNETWORKS_BAD_DATA; 5643d018d1a66872031568857fa369024c9640a085Miao Wang } 5762cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet state = ModelArgumentInfo::POINTER; 5862cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet } 59389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet buffer = data; 6062cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet locationAndLength = {.poolIndex = 0, .offset = 0, .length = length}; 61389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 62389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} 63389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet 64389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouilletint ModelArgumentInfo::setFromMemory(const Operand& operand, const ANeuralNetworksOperandType* type, 65389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t poolIndex, uint32_t offset, uint32_t length) { 66389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet int n = updateDimensionInfo(operand, type); 67389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 68389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return n; 69389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 7043d018d1a66872031568857fa369024c9640a085Miao Wang uint32_t neededLength = sizeOfData(operand.type, dimensions); 7143d018d1a66872031568857fa369024c9640a085Miao Wang if (operand.type != OperandType::OEM && neededLength != length) { 7243d018d1a66872031568857fa369024c9640a085Miao Wang LOG(ERROR) << "Setting argument with invalid length: " << length 7343d018d1a66872031568857fa369024c9640a085Miao Wang << ", expected length: " << neededLength; 7443d018d1a66872031568857fa369024c9640a085Miao Wang return ANEURALNETWORKS_BAD_DATA; 7543d018d1a66872031568857fa369024c9640a085Miao Wang } 7643d018d1a66872031568857fa369024c9640a085Miao Wang 77389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet state = ModelArgumentInfo::MEMORY; 7862cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet locationAndLength = {.poolIndex = poolIndex, .offset = offset, .length = length}; 79389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet buffer = nullptr; 80389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 81389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} 82389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet 838fb14e90ceb360adfbac0f708d27161b7c5b7fc5David Grossint ModelArgumentInfo::setFromTemporaryMemory(const Operand& operand, 848fb14e90ceb360adfbac0f708d27161b7c5b7fc5David Gross uint32_t poolIndex, uint32_t offset) { 8543d018d1a66872031568857fa369024c9640a085Miao Wang int n = updateDimensionInfo(operand, nullptr); 8643d018d1a66872031568857fa369024c9640a085Miao Wang if (n != ANEURALNETWORKS_NO_ERROR) { 8743d018d1a66872031568857fa369024c9640a085Miao Wang return n; 8843d018d1a66872031568857fa369024c9640a085Miao Wang } 8996811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross state = ModelArgumentInfo::MEMORY; 904d83c52f52613585f7b86368be762b2857f7460fDavid Gross locationAndLength = 918fb14e90ceb360adfbac0f708d27161b7c5b7fc5David Gross {.poolIndex = poolIndex, .offset = offset, .length = sizeOfData(operand)}; 9296811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross buffer = nullptr; 9396811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross return ANEURALNETWORKS_NO_ERROR; 9496811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross} 9596811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 96389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouilletint ModelArgumentInfo::updateDimensionInfo(const Operand& operand, 97389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet const ANeuralNetworksOperandType* newType) { 9843d018d1a66872031568857fa369024c9640a085Miao Wang nnAssert(dimensions.empty()); 99389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (newType == nullptr) { 10043d018d1a66872031568857fa369024c9640a085Miao Wang for (auto i : operand.dimensions) { 10143d018d1a66872031568857fa369024c9640a085Miao Wang if (i == 0) { 10243d018d1a66872031568857fa369024c9640a085Miao Wang LOG(ERROR) << "Setting input/output with unspecified dimensions"; 10343d018d1a66872031568857fa369024c9640a085Miao Wang return ANEURALNETWORKS_BAD_DATA; 10443d018d1a66872031568857fa369024c9640a085Miao Wang } 10543d018d1a66872031568857fa369024c9640a085Miao Wang } 10643d018d1a66872031568857fa369024c9640a085Miao Wang dimensions = operand.dimensions; 107389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } else { 108d2d0c031c43e8e5aafc75e8a652d79bcc2aaca99Jean-Luc Brouillet uint32_t count = newType->dimensionCount; 109389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (static_cast<OperandType>(newType->type) != operand.type || 110389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet count != operand.dimensions.size()) { 11143d018d1a66872031568857fa369024c9640a085Miao Wang LOG(ERROR) << "Setting input/output with incompatible types"; 112389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 113389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 11446a54e8a58e39c2fcb3448de04be76d95e744618Chenjie Luo 11546a54e8a58e39c2fcb3448de04be76d95e744618Chenjie Luo dimensions = hidl_vec<uint32_t>(count); 116389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet for (uint32_t i = 0; i < count; i++) { 11743d018d1a66872031568857fa369024c9640a085Miao Wang if (operand.dimensions[i] != 0 && operand.dimensions[i] != newType->dimensions[i]) { 11843d018d1a66872031568857fa369024c9640a085Miao Wang LOG(ERROR) << "Overriding a fully specified dimension is disallowed"; 11943d018d1a66872031568857fa369024c9640a085Miao Wang return ANEURALNETWORKS_BAD_DATA; 12043d018d1a66872031568857fa369024c9640a085Miao Wang } else { 12143d018d1a66872031568857fa369024c9640a085Miao Wang dimensions[i] = newType->dimensions[i]; 12243d018d1a66872031568857fa369024c9640a085Miao Wang } 123389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 124389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 125389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 126389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} 127389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet 1283ced3cfd5b8f22b632c35f24e585c4847383b195David GrossExecutionBuilder::ExecutionBuilder(const CompilationBuilder* compilation) : 12983e24dc4706a5b7089881a55daf05b3924fab3b7David Gross mModel(compilation->mModel), 1301f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mPlan(&compilation->mPlan), 131c2f1c1198c84f5a75fc2305935155f33b8ff5db2David Gross mPartitioning(compilation->mPartitioning), 13283e24dc4706a5b7089881a55daf05b3924fab3b7David Gross mInputs(mModel->inputCount()), 1331da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet mOutputs(mModel->outputCount()) { 134820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "ExecutionBuilder::ExecutionBuilder"; 135707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 136707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1373ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setInput(uint32_t index, const ANeuralNetworksOperandType* type, 138e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const void* buffer, size_t length) { 139707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mInputs.size()); 140707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (index >= count) { 1413ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setInput bad index " << index << " " << count; 142707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 143707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 144e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (type != nullptr) { 145e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet int n = validateOperandType(*type, "ANeuralNetworksExecution_setInput", false); 146e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 147e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return n; 148e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 149e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 150e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (length > 0xFFFFFFFF) { 151e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet LOG(ERROR) << "ANeuralNetworksExecution_setInput input exceeds max length " << length; 152e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 153e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 154e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet uint32_t l = static_cast<uint32_t>(length); 155389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return mInputs[index].setFromPointer(mModel->getInputOperand(index), type, 156e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const_cast<void*>(buffer), l); 157707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 158707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1593ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setInputFromMemory(uint32_t index, const ANeuralNetworksOperandType* type, 160e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const Memory* memory, size_t offset, size_t length) { 16196811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // Should be similar to StepExecutor::setInputOrOutputFromTemporaryMemory() 16296811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 1638b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mInputs.size()); 1648b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (index >= count) { 1653ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setInputFromMemory bad index " << index << " " 1668b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet << count; 1678b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 1688b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 169105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang if (!memory->validateSize(offset, length)) { 170105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang return ANEURALNETWORKS_BAD_DATA; 171105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang } 172e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet // TODO validate the rest 173389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t poolIndex = mMemories.add(memory); 174389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return mInputs[index].setFromMemory(mModel->getInputOperand(index), type, poolIndex, offset, 175389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet length); 176707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 177707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1783ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setOutput(uint32_t index, const ANeuralNetworksOperandType* type, void* buffer, 179e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet size_t length) { 180707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mOutputs.size()); 181707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (index >= count) { 1823ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setOutput bad index " << index << " " << count; 183707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 184707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 185e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (type != nullptr) { 186e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet int n = validateOperandType(*type, "ANeuralNetworksExecution_setOutput", false); 187e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 188e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return n; 189e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 190e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 191e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (length > 0xFFFFFFFF) { 192e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet LOG(ERROR) << "ANeuralNetworksExecution_setOutput input exceeds max length " << length; 193e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 194e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 195e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet uint32_t l = static_cast<uint32_t>(length); 196e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return mOutputs[index].setFromPointer(mModel->getOutputOperand(index), type, buffer, l); 197707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 198707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1993ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setOutputFromMemory(uint32_t index, const ANeuralNetworksOperandType* type, 200e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const Memory* memory, size_t offset, size_t length) { 20196811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // Should be similar to StepExecutor::setInputOrOutputFromTemporaryMemory() 20296811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 2038b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mOutputs.size()); 2048b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (index >= count) { 2053ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setOutputFromMemory bad index " << index << " " 2068b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet << count; 2078b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 2088b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 209105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang if (!memory->validateSize(offset, length)) { 210105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang return ANEURALNETWORKS_BAD_DATA; 211105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang } 212e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet // TODO validate the rest 213389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t poolIndex = mMemories.add(memory); 214389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return mOutputs[index].setFromMemory(mModel->getOutputOperand(index), type, poolIndex, offset, 215389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet length); 216707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 217707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 2185e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// Attempt synchronous execution of full model on CPU. 2195e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// Ensure that executionCallback->notify() is called. 2205e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Grossstatic void cpuFallbackFull(const ExecutionBuilder* executionBuilder, 2215e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const sp<ExecutionCallback>& executionCallback) { 222820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "cpuFallbackFull"; 2235e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross StepExecutor executor(executionBuilder, executionBuilder->getModel(), 22475886e77f9ca074173a49283b5c0a8c182d98977Michael Butler nullptr /* no VersionedIDevice, so CPU */, 2255e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross nullptr /* no IPreparedModel */); 2265e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executor.mapInputsAndOutputsTrivially(); 2275e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross sp<ExecutionCallback> fallbackCallback; 22807ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross int n = executor.startCompute(&fallbackCallback); 22907ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross if (n != ANEURALNETWORKS_NO_ERROR) { 23007ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross executionCallback->notify(convertResultCodeToErrorStatus(n)); 2315e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 2325e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2335e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross fallbackCallback->wait(); 2345e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->notify(fallbackCallback->getStatus()); 2355e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross} 2365e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross 2375e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// Attempt synchronous execution on CPU. 2385e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// (1) First, attempt to execute this step on CPU. If successful, 2395e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// return true. (Do not call executionCallback->notify().) 2405e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// (2) If unsuccessful, attempt to execute the full model on CPU, 2415e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// ensure that executionCallback->notify() is called, and return 2425e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross// false. 2435e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Grossstatic bool cpuFallbackPartial(const ExecutionBuilder* executionBuilder, 2445e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const ExecutionPlan* plan, 2455e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross std::shared_ptr<ExecutionPlan::Controller> controller, 2465e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const sp<ExecutionCallback>& executionCallback) { 247820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "cpuFallbackPartial"; 2485e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross std::shared_ptr<StepExecutor> executor; 2495e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross int n = plan->fallback(controller, &executor); 2505e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (n != ANEURALNETWORKS_NO_ERROR || executor->isCpu()) { 2515e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross cpuFallbackFull(executionBuilder, executionCallback); 2525e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return false; 2535e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2545e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross sp<ExecutionCallback> fallbackCallback; 2555e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (executor->startComputeOnCpu(&fallbackCallback) != ANEURALNETWORKS_NO_ERROR) { 2565e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross cpuFallbackFull(executionBuilder, executionCallback); 2575e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return false; 2585e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2595e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross fallbackCallback->wait(); 2605e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (fallbackCallback->getStatus() != ErrorStatus::NONE) { 2615e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross cpuFallbackFull(executionBuilder, executionCallback); 2625e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return false; 2635e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2645e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return true; 2655e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross} 2665e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross 2675e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Grossstatic void asyncStartComputePartitioned(const ExecutionBuilder* executionBuilder, 2685e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const ExecutionPlan* plan, 269e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross std::shared_ptr<ExecutionPlan::Controller> controller, 2705e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross bool allowFallback, 2715e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const sp<ExecutionCallback>& executionCallback) { 272820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "ExecutionBuilder::startCompute (from plan, iteratively)"; 273e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross while (true) { 274e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross std::shared_ptr<StepExecutor> executor; 275820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "looking for next StepExecutor"; 276e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross int n = plan->next(controller, &executor); 2775e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (n != ANEURALNETWORKS_NO_ERROR) { 2785e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (allowFallback) { 2795e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross cpuFallbackFull(executionBuilder, executionCallback); 2805e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 28107ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross executionCallback->notify(convertResultCodeToErrorStatus(n)); 2825e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2835e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 2845e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 2855e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (executor == nullptr) { 2865e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->notify(ErrorStatus::NONE); 287e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross return; 288e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross } 289e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross 290e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross sp<ExecutionCallback> stepCallback; 291e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross n = executor->startCompute(&stepCallback); 292e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross if (n != ANEURALNETWORKS_NO_ERROR) { 2935e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (allowFallback) { 2945e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (cpuFallbackPartial(executionBuilder, plan, controller, executionCallback)) { 2955e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Successfully executed one step on CPU. 2965e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross continue; 2975e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 2985e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Either successfully executed entire plan on 2995e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // CPU, or tried and failed to do so. 3005e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 3015e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 3025e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 30307ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross executionCallback->notify(convertResultCodeToErrorStatus(n)); 3045e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 3055e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 306e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross } 307e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross stepCallback->wait(); 308e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross ErrorStatus status = stepCallback->getStatus(); 309e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross if (status != ErrorStatus::NONE) { 3105e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (allowFallback) { 3115e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (cpuFallbackPartial(executionBuilder, plan, controller, executionCallback)) { 3125e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Successfully executed one step on CPU. 3135e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross continue; 3145e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 3155e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Either successfully executed entire plan on 3165e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // CPU, or tried and failed to do so. 3175e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 3185e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 3195e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 3205e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->notify(status); 3215e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return; 3225e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 323e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross } 324e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross } 325e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross} 326e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross 327033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint ExecutionBuilder::startCompute(sp<ExecutionCallback>* synchronizationCallback) { 328033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = nullptr; 329425b2594c76e934dfdbc93209253e3c189571149David Gross 330707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // TODO validate that we have full types for all inputs and outputs, 331707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // that the graph is not cyclic, 332f1817c663af4f22bc089ef82cd50df4186422c42Yang Ni 3338b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& p : mInputs) { 334389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (p.state == ModelArgumentInfo::UNSPECIFIED) { 3353ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_startCompute not all inputs specified"; 336707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 337707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 338707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 3398b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& p : mOutputs) { 340389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (p.state == ModelArgumentInfo::UNSPECIFIED) { 3413ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_startCompute not all outputs specified"; 342707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 343707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 344707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 345707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 3465e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross#ifndef DISABLE_PARTITIONED_EXECUTION 3475e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross { 3485e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // TODO: Remove the non-plan-based path once we've fully integrated ExecutionPlan 3495e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // with the compilation and execution phases of the NN API? Or retain that path 3505e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // as a fallback in the case of partitioning failure? 3515e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // 3525e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // TODO: Entire plan-based-path should run in an asynchronous thread -- 3535e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // take the asynchronous thread logic out of startComputeOnCpu() and use 3545e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // it to wrap the plan-based-path. 355c2f1c1198c84f5a75fc2305935155f33b8ff5db2David Gross if (mPartitioning > 0) { 356c2f1c1198c84f5a75fc2305935155f33b8ff5db2David Gross const bool allowFallback = DeviceManager::partitioningAllowsFallback(mPartitioning); 3575e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross std::shared_ptr<ExecutionPlan::Controller> controller = mPlan->makeController(this); 3585e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (controller == nullptr) { 3595e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (!allowFallback) { 3605e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return ANEURALNETWORKS_OP_FAILED; 3615e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 3625e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } else { 3635e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // TODO: use a thread pool 364b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 3655e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Prepare the callback for asynchronous execution. 3665e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // sp<ExecutionCallback> object is returned when the 3675e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // execution has been successfully launched, otherwise a 3685e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // nullptr is returned. The executionCallback is 3695e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // abstracted in the NN API as an "event". 3705e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross sp<ExecutionCallback> executionCallback = new ExecutionCallback(); 3715e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross std::thread thread(asyncStartComputePartitioned, this, mPlan, controller, 3725e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross allowFallback, 3735e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback); 3745e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executionCallback->bind_thread(std::move(thread)); 3755e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross *synchronizationCallback = executionCallback; 3765e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return ANEURALNETWORKS_NO_ERROR; 3775e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 3781f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross } 3791f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross } 3805e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross#else 3815e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross { 3825e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Find a driver that can handle all the operations. 3835e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // TODO: Does not handle CPU fallback (which is tricky because 3845e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // StepExecutor::startCompute() is designed as 3855e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // asynchronous). 3865e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // TODO: Does not actually behave asynchronously (because 3875e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // StepExecutor::startCompute() isn't actually asynchronous 3885e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // on a device as opposed to a CPU). 3895e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross Model hidlModel; 3905e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross mModel->setHidlModel(&hidlModel); 3915e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross const std::vector<std::shared_ptr<Device>>& devices = DeviceManager::get()->getDrivers(); 3925e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross for (const auto& device : devices) { 3935e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross hidl_vec<bool> supports; 394820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "Checking " << device->getName(); 3955e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross device->getSupportedOperations(hidlModel, &supports); 3965e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross if (std::find(supports.begin(), supports.end(), false) == supports.end()) { 397820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "ExecutionBuilder::startCompute (without plan) on " << device->getName(); 3985e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross StepExecutor executor(this, mModel, device->getInterface(), 3995e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross nullptr /* no IPreparedModel, so compile */); 4005e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross executor.mapInputsAndOutputsTrivially(); 4015e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross return executor.startCompute(synchronizationCallback); 4025e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross } 403ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet } 404ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet } 4055e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross#endif // DISABLE_PARTITIONED_EXECUTION 4065e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross 4075e8feed5e8a07bab1ec395e5a01bb8900db00cecDavid Gross // Run on the CPU. 408820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "ExecutionBuilder::startCompute (without plan) on CPU"; 409b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross StepExecutor executor(this, mModel, 41075886e77f9ca074173a49283b5c0a8c182d98977Michael Butler nullptr /* no VersionedIDevice, so CPU */, 411b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross nullptr /* no IPreparedModel */); 412b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross executor.mapInputsAndOutputsTrivially(); 413033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return executor.startCompute(synchronizationCallback); 414707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 415707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 416707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet// Figures out how to place each of the input or outputs in a buffer. This just does the layout, 4178b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet// it does not copy data. Aligns each input a bit. 418b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Grossint StepExecutor::allocatePointerArgumentsToPool(std::vector<ModelArgumentInfo>* args, 419b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross Memory* memory) { 4208b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t nextPoolIndex = mMemories.size(); 421707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet int64_t total = 0; 4228b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& info : *args) { 4238b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (info.state == ModelArgumentInfo::POINTER) { 42462cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet DataLocation& loc = info.locationAndLength; 4258b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // TODO Good enough alignment? 4268b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet total += alignBytesNeeded(static_cast<uint32_t>(total), loc.length); 4278b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet loc.poolIndex = nextPoolIndex; 4288b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet loc.offset = static_cast<uint32_t>(total); 4298b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet total += loc.length; 4308b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 4318b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet }; 432707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (total > 0xFFFFFFFF) { 4333ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_startCompute Size of all inputs or outputs exceeds " 434707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet "2^32."; 435707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 436707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 4378b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet hidl_memory hidlMemory; 4388b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (total > 0) { 439389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet memory->create(total); // TODO check error 4408b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet mMemories.add(memory); 4418b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 442707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 443707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 444707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 44562cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouilletstatic void setRequestArgumentArray(const std::vector<ModelArgumentInfo>& argumentInfos, 446a4e2ee81d015b5e26897d18b18dd50044faba62fJean-Luc Brouillet hidl_vec<RequestArgument>* ioInfos) { 4478b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet size_t count = argumentInfos.size(); 4488b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet ioInfos->resize(count); 4498b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (size_t i = 0; i < count; i++) { 45062cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet const auto& info = argumentInfos[i]; 45162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet (*ioInfos)[i] = { .hasNoValue = info.state == ModelArgumentInfo::HAS_NO_VALUE, 45262cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet .location = info.locationAndLength, 45362cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet .dimensions = info.dimensions, 45462cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet }; 4558b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 4568b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet} 4578b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet 458b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David GrossStepExecutor::StepExecutor(const ExecutionBuilder* executionBuilder, 459b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross const ModelBuilder* model, 46075886e77f9ca074173a49283b5c0a8c182d98977Michael Butler VersionedIDevice* driver, sp<IPreparedModel> preparedModel) : 461b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mExecutionBuilder(executionBuilder), mModel(model), 462891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross mDriver(driver), mPreparedModel(preparedModel), 463891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross mInputs(model->inputCount()), mOutputs(model->outputCount()) {} 464b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 465b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Grossvoid StepExecutor::mapInputsAndOutputsTrivially() { 466b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mInputs = mExecutionBuilder->mInputs; 467b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mOutputs = mExecutionBuilder->mOutputs; 468b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mMemories = mExecutionBuilder->mMemories; 469b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross} 470b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 471891b10f7048c62a37a74c4b570be220089dfd55eDavid Grossvoid StepExecutor::mapInputOrOutput(const ModelArgumentInfo& builderInputOrOutput, 472891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross ModelArgumentInfo* executorInputOrOutput) { 473891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross *executorInputOrOutput = builderInputOrOutput; 474891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross switch (executorInputOrOutput->state) { 475891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross default: 476891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross nnAssert(!"unexpected ModelArgumentInfo::state"); 477891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross case ModelArgumentInfo::POINTER: 478891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross case ModelArgumentInfo::UNSPECIFIED: 479891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross break; 480891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross case ModelArgumentInfo::MEMORY: { 481891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const uint32_t builderPoolIndex = 48262cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet builderInputOrOutput.locationAndLength.poolIndex; 483891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const Memory* memory = mExecutionBuilder->mMemories[builderPoolIndex]; 484891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const uint32_t executorPoolIndex = mMemories.add(memory); 48562cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet executorInputOrOutput->locationAndLength.poolIndex = 486891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross executorPoolIndex; 487891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross break; 488891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 489891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 490891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross} 491891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross 49296811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Grossint StepExecutor::setInputOrOutputFromTemporaryMemory(const Operand& inputOrOutputOperand, 4938fb14e90ceb360adfbac0f708d27161b7c5b7fc5David Gross const Memory* memory, uint32_t offset, 49496811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross ModelArgumentInfo* inputOrOutputInfo) { 49596811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // Should be similar to 49696811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // ExecutionBuilder::setInputFromMemory() 49796811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // ExecutionBuilder::setOutputFromMemory() 49896811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 49996811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross uint32_t poolIndex = mMemories.add(memory); 5008fb14e90ceb360adfbac0f708d27161b7c5b7fc5David Gross return inputOrOutputInfo->setFromTemporaryMemory(inputOrOutputOperand, poolIndex, offset); 50196811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross} 50296811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 503c4c264098a728268ad28084ea6e0263d9c1d7868David Grossstatic void logArguments(const char* kind, const std::vector<ModelArgumentInfo> &args) { 504c4c264098a728268ad28084ea6e0263d9c1d7868David Gross for (unsigned i = 0; i < args.size(); i++) { 505c4c264098a728268ad28084ea6e0263d9c1d7868David Gross const auto& arg = args[i]; 506c4c264098a728268ad28084ea6e0263d9c1d7868David Gross std::string prefix = kind + std::string("[") + std::to_string(i) + "] = "; 507c4c264098a728268ad28084ea6e0263d9c1d7868David Gross switch (arg.state) { 508c4c264098a728268ad28084ea6e0263d9c1d7868David Gross case ModelArgumentInfo::POINTER: 50964031fa26110e6d57896bfdbe1cd58e69cde3dbbMiao Wang VLOG(EXECUTION) << prefix << "POINTER(" << SHOW_IF_DEBUG(arg.buffer) << ")"; 510c4c264098a728268ad28084ea6e0263d9c1d7868David Gross break; 511c4c264098a728268ad28084ea6e0263d9c1d7868David Gross case ModelArgumentInfo::MEMORY: 512c4c264098a728268ad28084ea6e0263d9c1d7868David Gross VLOG(EXECUTION) << prefix << "MEMORY(" 513c4c264098a728268ad28084ea6e0263d9c1d7868David Gross << "pool=" << arg.locationAndLength.poolIndex 514c4c264098a728268ad28084ea6e0263d9c1d7868David Gross << ", " 515c4c264098a728268ad28084ea6e0263d9c1d7868David Gross << "off=" << arg.locationAndLength.offset 516c4c264098a728268ad28084ea6e0263d9c1d7868David Gross << ")"; 517c4c264098a728268ad28084ea6e0263d9c1d7868David Gross break; 518c4c264098a728268ad28084ea6e0263d9c1d7868David Gross case ModelArgumentInfo::HAS_NO_VALUE: 519c4c264098a728268ad28084ea6e0263d9c1d7868David Gross VLOG(EXECUTION) << prefix << "HAS_NO_VALUE"; 520c4c264098a728268ad28084ea6e0263d9c1d7868David Gross break; 521c4c264098a728268ad28084ea6e0263d9c1d7868David Gross case ModelArgumentInfo::UNSPECIFIED: 522c4c264098a728268ad28084ea6e0263d9c1d7868David Gross VLOG(EXECUTION) << prefix << "UNSPECIFIED"; 523c4c264098a728268ad28084ea6e0263d9c1d7868David Gross break; 524c4c264098a728268ad28084ea6e0263d9c1d7868David Gross default: 525c4c264098a728268ad28084ea6e0263d9c1d7868David Gross VLOG(EXECUTION) << prefix << "state(" << arg.state << ")"; 526c4c264098a728268ad28084ea6e0263d9c1d7868David Gross break; 527c4c264098a728268ad28084ea6e0263d9c1d7868David Gross } 528c4c264098a728268ad28084ea6e0263d9c1d7868David Gross } 529c4c264098a728268ad28084ea6e0263d9c1d7868David Gross} 530c4c264098a728268ad28084ea6e0263d9c1d7868David Gross 531033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint StepExecutor::startCompute(sp<ExecutionCallback>* synchronizationCallback) { 532c4c264098a728268ad28084ea6e0263d9c1d7868David Gross if (VLOG_IS_ON(EXECUTION)) { 533c4c264098a728268ad28084ea6e0263d9c1d7868David Gross logArguments("input", mInputs); 534c4c264098a728268ad28084ea6e0263d9c1d7868David Gross logArguments("output", mOutputs); 535c4c264098a728268ad28084ea6e0263d9c1d7868David Gross } 536b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross if (mDriver == nullptr) { 537033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return startComputeOnCpu(synchronizationCallback); 538b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross } else { 539033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return startComputeOnDevice(synchronizationCallback); 540b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross } 541b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross} 542b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 543033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint StepExecutor::startComputeOnDevice(sp<ExecutionCallback>* synchronizationCallback) { 544b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross nnAssert(mDriver != nullptr); 545b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 546033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = nullptr; 547425b2594c76e934dfdbc93209253e3c189571149David Gross 548b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross // TODO: Remove the mPreparedModel == nullptr case once we've fully integrated 5491f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // ExecutionPlan with the compilation and execution phases of the NN API 550b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross if (mPreparedModel == nullptr) { 5511f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross Model model; 5521f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mModel->setHidlModel(&model); 5531f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross 5541f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // TODO Dangerous! In async, the model will outlive it here. Safe for now 555033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback(); 5561e9666208595bc251a8958155b1e41eca90b69dbMichael Butler // TODO(butlermichael): Propagate user preference to this point instead of 5571e9666208595bc251a8958155b1e41eca90b69dbMichael Butler // using default value of ANEURALNETWORKS_PREFER_FAST_SINGLE_ANSWER, or 5581e9666208595bc251a8958155b1e41eca90b69dbMichael Butler // remove this entire block of code since it is a stale path that is only 5591e9666208595bc251a8958155b1e41eca90b69dbMichael Butler // encountered on an #if-removed code. 5601e9666208595bc251a8958155b1e41eca90b69dbMichael Butler ExecutionPreference preference = 5611e9666208595bc251a8958155b1e41eca90b69dbMichael Butler static_cast<ExecutionPreference>(ANEURALNETWORKS_PREFER_FAST_SINGLE_ANSWER); 5621e9666208595bc251a8958155b1e41eca90b69dbMichael Butler ErrorStatus prepareLaunchStatus = mDriver->prepareModel(model, preference, 5631e9666208595bc251a8958155b1e41eca90b69dbMichael Butler preparedModelCallback); 56475886e77f9ca074173a49283b5c0a8c182d98977Michael Butler if (prepareLaunchStatus != ErrorStatus::NONE) { 56507ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross return convertErrorStatusToResultCode(prepareLaunchStatus); 566033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler } 5675f916fc9a7ae95f172492bdfe5344c37beff3a6fMichael Butler 568033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // Immediately synchronize with callback object for now 5691f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // TODO: change to asynchronous later 570033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler preparedModelCallback->wait(); 571033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); 572033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler mPreparedModel = preparedModelCallback->getPreparedModel(); 57307ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross if (prepareReturnStatus != ErrorStatus::NONE) { 57407ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross return convertErrorStatusToResultCode(prepareReturnStatus); 57507ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross } 57607ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross if (mPreparedModel == nullptr) { 5771f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross return ANEURALNETWORKS_OP_FAILED; 5781f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross } 579707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 580707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 581e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // We separate the input & output pools so that we reduce the copying done if we 582e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // do an eventual remoting (hidl_memory->update()). We could also use it to set 583e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // protection on read only memory but that's not currently done. 584e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross Memory inputPointerArguments; 585e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross Memory outputPointerArguments; 586e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross 5878b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // Layout the input and output data 588e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross int n = allocatePointerArgumentsToPool(&mInputs, &inputPointerArguments); 589707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 590707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return n; 591707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 592e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross n = allocatePointerArgumentsToPool(&mOutputs, &outputPointerArguments); 593707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 594707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return n; 595707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 596707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 5978b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // Copy the input data that was specified via a pointer. 598e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // inputPointerArguments.update(); 5998b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& info : mInputs) { 6008b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (info.state == ModelArgumentInfo::POINTER) { 60162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet DataLocation& loc = info.locationAndLength; 6022150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet uint8_t* data = nullptr; 603e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross int n = inputPointerArguments.getPointer(&data); 6042150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 6052150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet return n; 6062150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet } 6078b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet memcpy(data + loc.offset, info.buffer, loc.length); 6088b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 609707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 610e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // TODO: Add inputPointerArguments.commit() and .update() at all the right places 611707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 612707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet Request request; 61362cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mInputs, &request.inputs); 61462cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mOutputs, &request.outputs); 6158b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = mMemories.size(); 6168b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet request.pools.resize(count); 6178b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (uint32_t i = 0; i < count; i++) { 6188b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet request.pools[i] = mMemories[i]->getHidlMemory(); 6198b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 620707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 621033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // Prepare the callback for asynchronous execution. sp<ExecutionCallback> 622033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // object is returned when the execution has been successfully launched, 623033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // otherwise a nullptr is returned. The executionCallback is abstracted in 624033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // the NN API as an "event". 625033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // 626033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // The sp is used for ref-counting purposes. Without it, the HIDL service 627033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // could attempt to communicate with a dead callback object. 628689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // 629033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // TODO: Explain the "dead callback" problem further, either here or 630689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // in the design document. 631033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler sp<ExecutionCallback> executionCallback = new ExecutionCallback(); 632689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 63364031fa26110e6d57896bfdbe1cd58e69cde3dbbMiao Wang VLOG(EXECUTION) << "Before mPreparedModel->execute() " << SHOW_IF_DEBUG(toString(request)); 6343ced3cfd5b8f22b632c35f24e585c4847383b195David Gross // Execute. 635033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // TODO: What happens to the Callback if the service dies abnormally 636033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // -- won't that keep the Callback live forever, because the service 637689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // never has the opportunity to bump the reference count down? Or 638689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // maybe the HIDL infrastructure handles this magically? At worst, 639033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // it seems like this is a small memory leak, if the Callback stays 640689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // alive forever. 64106b5a0a85abe10b618a3a5ed023ac5c74678647eDavid Gross Return<ErrorStatus> executeStatus = mPreparedModel->execute(request, executionCallback); 64206b5a0a85abe10b618a3a5ed023ac5c74678647eDavid Gross if (!executeStatus.isOk() || executeStatus != ErrorStatus::NONE) { 643820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "**Execute failed**"; 64407ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross return executeStatus.isOk() 64507ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross ? convertErrorStatusToResultCode(executeStatus) 64607ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross : ANEURALNETWORKS_OP_FAILED; 647707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 648707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 649689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: Remove this synchronization point when the block of code below is 650689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // removed. 651033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler executionCallback->wait(); 65206b5a0a85abe10b618a3a5ed023ac5c74678647eDavid Gross Return<ErrorStatus> callbackStatus = executionCallback->getStatus(); 65306b5a0a85abe10b618a3a5ed023ac5c74678647eDavid Gross if (!callbackStatus.isOk() || callbackStatus != ErrorStatus::NONE) { 654820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "**Execute async failed**"; 65507ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross return callbackStatus.isOk() 65607ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross ? convertErrorStatusToResultCode(callbackStatus) 65707ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross : ANEURALNETWORKS_OP_FAILED; 658689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler } 659689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 660707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // Copy the output data from shared memory to the output buffers. 661689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: Move this block of code somewhere else. It should not be in the 662689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // startCompute function. 663689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: outputMemory->update(); outputMemory->commit() 6648b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& info : mOutputs) { 6658b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (info.state == ModelArgumentInfo::POINTER) { 66662cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet DataLocation& loc = info.locationAndLength; 6672150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet uint8_t* data = nullptr; 668e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross int n = outputPointerArguments.getPointer(&data); 6692150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 6702150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet return n; 6712150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet } 6728b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet memcpy(info.buffer, data + loc.offset, loc.length); 6738b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 674707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 675820215d28bed6c90f696cde0f282445d16da432eMiao Wang VLOG(EXECUTION) << "StepExecutor::startComputeOnDevice completed"; 676707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 677033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = executionCallback; 678707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 679707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 680707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 681689d892203c06c66c7bb2e374462a8434e40b75fMichael Butlerstatic void asyncStartComputeOnCpu(const Model& model, const Request& request, 6821da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet const std::vector<RunTimePoolInfo>& modelPoolInfos, 6831da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet const std::vector<RunTimePoolInfo>& requestPoolInfos, 684033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler const sp<IExecutionCallback>& executionCallback) { 685689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler CpuExecutor executor; 6861da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet int err = executor.run(model, request, modelPoolInfos, requestPoolInfos); 68707ed4d56a34eb1023c308f40ad7a0a1d07816642David Gross executionCallback->notify(convertResultCodeToErrorStatus(err)); 688689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler} 689689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 690033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint StepExecutor::startComputeOnCpu(sp<ExecutionCallback>* synchronizationCallback) { 691707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // TODO: use a thread pool 692689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 6931f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross Model model; 6941f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mModel->setHidlModel(&model); 6951f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross 696033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // Prepare the callback for asynchronous execution. sp<ExecutionCallback> 697033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // object is returned when the execution has been successfully launched, 698033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // otherwise a nullptr is returned. The executionCallback is abstracted in 699033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // the NN API as an "event". 700033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler sp<ExecutionCallback> executionCallback = new ExecutionCallback(); 701033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = nullptr; 702707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 7031da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet std::vector<RunTimePoolInfo> modelPoolInfos; 7041da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet if (!setRunTimePoolInfosFromHidlMemories(&modelPoolInfos, model.pools)) { 7051da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet return ANEURALNETWORKS_UNMAPPABLE; 7061da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet } 7071da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet 7081da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet std::vector<RunTimePoolInfo> requestPoolInfos; 7094cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross requestPoolInfos.reserve(mMemories.size()); 7104cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross bool fail = false; 7114cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross for (const Memory* mem : mMemories) { 7124cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross requestPoolInfos.emplace_back(mem->getHidlMemory(), &fail); 7134cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross } 7144cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross if (fail) { 7154cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross return ANEURALNETWORKS_UNMAPPABLE; 7168b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 7178b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // Create as many pools as there are input / output. 7181da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet auto fixPointerArguments = [&requestPoolInfos](std::vector<ModelArgumentInfo>& argumentInfos) { 7198b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (ModelArgumentInfo& argumentInfo : argumentInfos) { 7208b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (argumentInfo.state == ModelArgumentInfo::POINTER) { 72162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet argumentInfo.locationAndLength.poolIndex = 7221da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet static_cast<uint32_t>(requestPoolInfos.size()); 72362cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet argumentInfo.locationAndLength.offset = 0; 7244cb4fa2fe863a8f404c0448da160dd5bb4187db4David Gross requestPoolInfos.emplace_back(static_cast<uint8_t*>(argumentInfo.buffer)); 7258b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 7268b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 7278b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet }; 7288b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet fixPointerArguments(mInputs); 7298b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet fixPointerArguments(mOutputs); 730707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 7318b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet Request request; 73262cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mInputs, &request.inputs); 73362cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mOutputs, &request.outputs); 734707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 735689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: should model be moved with a std::cref? 736689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler std::thread thread(asyncStartComputeOnCpu, model, std::move(request), 7371da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet std::move(modelPoolInfos), std::move(requestPoolInfos), 7381da8fed77c5c296afa18f754ec3616e7f02a4cfdJean-Luc Brouillet executionCallback); 739033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler executionCallback->bind_thread(std::move(thread)); 740689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 741033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = executionCallback; 742689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler return ANEURALNETWORKS_NO_ERROR; 743707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 744707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 745389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} // namespace nn 746389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} // namespace android 747