ExecutionBuilder.cpp revision 96811e2b1347889a25bd9686f47ca3cbf061fb1b
1707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet/* 2707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * Copyright (C) 2017 The Android Open Source Project 3707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * 4707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * Licensed under the Apache License, Version 2.0 (the "License"); 5707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * you may not use this file except in compliance with the License. 6707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * You may obtain a copy of the License at 7707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * 8707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * http://www.apache.org/licenses/LICENSE-2.0 9707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * 10707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * Unless required by applicable law or agreed to in writing, software 11707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * distributed under the License is distributed on an "AS IS" BASIS, 12707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * See the License for the specific language governing permissions and 14707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * limitations under the License. 15707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet */ 16707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 173ced3cfd5b8f22b632c35f24e585c4847383b195David Gross#define LOG_TAG "ExecutionBuilder" 18707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 193ced3cfd5b8f22b632c35f24e585c4847383b195David Gross#include "ExecutionBuilder.h" 20707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 2183e24dc4706a5b7089881a55daf05b3924fab3b7David Gross#include "CompilationBuilder.h" 22707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "CpuExecutor.h" 23707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "HalInterfaces.h" 24707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "Manager.h" 25707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "ModelBuilder.h" 26707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 27689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler#include <mutex> 28389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet#include <thread> 29689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler#include <vector> 30689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 31707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouilletnamespace android { 32707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouilletnamespace nn { 33707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 34389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouilletint ModelArgumentInfo::setFromPointer(const Operand& operand, 35389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet const ANeuralNetworksOperandType* type, void* data, 36389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t length) { 37389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet int n = updateDimensionInfo(operand, type); 38389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 39389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return n; 40389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 41389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet state = ModelArgumentInfo::POINTER; 42a4e2ee81d015b5e26897d18b18dd50044faba62fJean-Luc Brouillet locationAndDimension.location = {.poolIndex = 0, .offset = 0, .length = length}; 43389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet buffer = data; 44389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 45389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} 46389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet 47389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouilletint ModelArgumentInfo::setFromMemory(const Operand& operand, const ANeuralNetworksOperandType* type, 48389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t poolIndex, uint32_t offset, uint32_t length) { 49389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet int n = updateDimensionInfo(operand, type); 50389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 51389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return n; 52389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 53389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet state = ModelArgumentInfo::MEMORY; 54389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet locationAndDimension.location = {.poolIndex = poolIndex, .offset = offset, .length = length}; 55389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet buffer = nullptr; 56389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 57389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} 58389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet 5996811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Grossint ModelArgumentInfo::setFromTemporaryMemory(const Operand& operand, uint32_t poolIndex) { 6096811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross locationAndDimension.dimensions = operand.dimensions; 6196811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross state = ModelArgumentInfo::MEMORY; 6296811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross locationAndDimension.location = 6396811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross {.poolIndex = poolIndex, .offset = 0, .length = sizeOfData(operand)}; 6496811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross buffer = nullptr; 6596811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross return ANEURALNETWORKS_NO_ERROR; 6696811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross} 6796811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 68389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouilletint ModelArgumentInfo::updateDimensionInfo(const Operand& operand, 69389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet const ANeuralNetworksOperandType* newType) { 70389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (newType == nullptr) { 71389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet locationAndDimension.dimensions = hidl_vec<uint32_t>(); 72389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } else { 73d2d0c031c43e8e5aafc75e8a652d79bcc2aaca99Jean-Luc Brouillet uint32_t count = newType->dimensionCount; 74389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (static_cast<OperandType>(newType->type) != operand.type || 75389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet count != operand.dimensions.size()) { 763ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setInput/Output incompatible types"; 77389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 78389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 79389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet for (uint32_t i = 0; i < count; i++) { 80d2d0c031c43e8e5aafc75e8a652d79bcc2aaca99Jean-Luc Brouillet locationAndDimension.dimensions[i] = newType->dimensions[i]; 81389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 82389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 83389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 84389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} 85389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet 863ced3cfd5b8f22b632c35f24e585c4847383b195David GrossExecutionBuilder::ExecutionBuilder(const CompilationBuilder* compilation) : 8783e24dc4706a5b7089881a55daf05b3924fab3b7David Gross mModel(compilation->mModel), 881f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mPlan(&compilation->mPlan), 8983e24dc4706a5b7089881a55daf05b3924fab3b7David Gross mInputs(mModel->inputCount()), 9083e24dc4706a5b7089881a55daf05b3924fab3b7David Gross mOutputs(mModel->outputCount()), 9183e24dc4706a5b7089881a55daf05b3924fab3b7David Gross mMemories(mModel->getMemories()) { 923ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(DEBUG) << "ExecutionBuilder::ExecutionBuilder"; 93707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 94707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 953ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setInput(uint32_t index, const ANeuralNetworksOperandType* type, 96e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const void* buffer, size_t length) { 97707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mInputs.size()); 98707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (index >= count) { 993ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setInput bad index " << index << " " << count; 100707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 101707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 102e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (type != nullptr) { 103e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet int n = validateOperandType(*type, "ANeuralNetworksExecution_setInput", false); 104e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 105e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return n; 106e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 107e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 108e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (length > 0xFFFFFFFF) { 109e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet LOG(ERROR) << "ANeuralNetworksExecution_setInput input exceeds max length " << length; 110e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 111e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 112e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet uint32_t l = static_cast<uint32_t>(length); 113389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return mInputs[index].setFromPointer(mModel->getInputOperand(index), type, 114e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const_cast<void*>(buffer), l); 115707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 116707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1173ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setInputFromMemory(uint32_t index, const ANeuralNetworksOperandType* type, 118e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const Memory* memory, size_t offset, size_t length) { 11996811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // Should be similar to StepExecutor::setInputOrOutputFromTemporaryMemory() 12096811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 1218b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mInputs.size()); 1228b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (index >= count) { 1233ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setInputFromMemory bad index " << index << " " 1248b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet << count; 1258b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 1268b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 127105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang if (!memory->validateSize(offset, length)) { 128105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang return ANEURALNETWORKS_BAD_DATA; 129105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang } 130e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet // TODO validate the rest 131389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t poolIndex = mMemories.add(memory); 132389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return mInputs[index].setFromMemory(mModel->getInputOperand(index), type, poolIndex, offset, 133389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet length); 134707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 135707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1363ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setOutput(uint32_t index, const ANeuralNetworksOperandType* type, void* buffer, 137e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet size_t length) { 138707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mOutputs.size()); 139707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (index >= count) { 1403ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setOutput bad index " << index << " " << count; 141707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 142707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 143e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (type != nullptr) { 144e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet int n = validateOperandType(*type, "ANeuralNetworksExecution_setOutput", false); 145e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 146e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return n; 147e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 148e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 149e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (length > 0xFFFFFFFF) { 150e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet LOG(ERROR) << "ANeuralNetworksExecution_setOutput input exceeds max length " << length; 151e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 152e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 153e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet uint32_t l = static_cast<uint32_t>(length); 154e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return mOutputs[index].setFromPointer(mModel->getOutputOperand(index), type, buffer, l); 155707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 156707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1573ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setOutputFromMemory(uint32_t index, const ANeuralNetworksOperandType* type, 158e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const Memory* memory, size_t offset, size_t length) { 15996811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // Should be similar to StepExecutor::setInputOrOutputFromTemporaryMemory() 16096811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 1618b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mOutputs.size()); 1628b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (index >= count) { 1633ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setOutputFromMemory bad index " << index << " " 1648b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet << count; 1658b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 1668b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 167105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang if (!memory->validateSize(offset, length)) { 168105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang return ANEURALNETWORKS_BAD_DATA; 169105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang } 170e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet // TODO validate the rest 171389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t poolIndex = mMemories.add(memory); 172389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return mOutputs[index].setFromMemory(mModel->getOutputOperand(index), type, poolIndex, offset, 173389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet length); 174707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 175707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 176033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint ExecutionBuilder::startCompute(sp<ExecutionCallback>* synchronizationCallback) { 177033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = nullptr; 178425b2594c76e934dfdbc93209253e3c189571149David Gross 179707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // TODO validate that we have full types for all inputs and outputs, 180707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // that the graph is not cyclic, 181f1817c663af4f22bc089ef82cd50df4186422c42Yang Ni /* 182f1817c663af4f22bc089ef82cd50df4186422c42Yang Ni TODO: For non-optional inputs, also verify that buffers are not null. 183f1817c663af4f22bc089ef82cd50df4186422c42Yang Ni 1848b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& p : mInputs) { 185389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (p.state == ModelArgumentInfo::UNSPECIFIED) { 1863ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_startCompute not all inputs specified"; 187707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 188707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 189707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 190f1817c663af4f22bc089ef82cd50df4186422c42Yang Ni */ 1918b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& p : mOutputs) { 192389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (p.state == ModelArgumentInfo::UNSPECIFIED) { 1933ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_startCompute not all outputs specified"; 194707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 195707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 196707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 197707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1981f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // TODO: Remove the non-plan-based path once we've fully integrated ExecutionPlan 1991f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // with the compilation and execution phases of the NN API. 200b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross // 201b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross // TODO: Entire plan-based-path should run in an asynchronous thread -- 202b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross // take the asynchronous thread logic out of startComputeOnCpu() and use 203b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross // it to wrap the plan-based-path. 2041f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross#if NN_DEBUGGABLE 205891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross { 206891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const int partitioning = DeviceManager::get()->getPartitioning(); 207891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross if (partitioning > 0) { 208891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const bool simulation = !((partitioning > 1) && mPlan->shouldBeExecutable()); 209891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross LOG(DEBUG) << "ExecutionBuilder::startCompute" 210891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross << (simulation ? " SIMULATION" : "") 211891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross << " (from plan, iteratively)"; 212891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross ExecutionPlan::Controller controller = mPlan->makeController(this); 213891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross while (true) { 214891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross LOG(DEBUG) << "looking for next StepExecutor"; 215891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross std::shared_ptr<StepExecutor> executor; 216891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross int n = mPlan->next(&controller, &executor); 217891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross if (n != ANEURALNETWORKS_NO_ERROR || executor == nullptr) { 218891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross if (!simulation) { 219891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross return n; 220891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 221b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 222891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross // simulation 223891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross if (n != ANEURALNETWORKS_NO_ERROR) { 224891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross LOG(DEBUG) << "ExecutionBuilder::startCompute SIMULATION failed " 225891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross << "with error " << n; 226891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 227891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross break; 228891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 229891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross if (simulation) { 230891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross continue; 231891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 232891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross 233033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler n = executor->startCompute(synchronizationCallback); 234891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross if (n != ANEURALNETWORKS_NO_ERROR) { 235891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross return n; 236891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 237033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler (*synchronizationCallback)->wait(); 238033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler if ((*synchronizationCallback)->getStatus() != ErrorStatus::NONE) { 239891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross return ANEURALNETWORKS_OP_FAILED; 240891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 241b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross } 2421f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross } 2431f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross } 2441f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross#endif // NN_DEBUGGABLE 245ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet 246ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet // Find a driver that can handle all the operations. 2471f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross Model hidlModel; 2481f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mModel->setHidlModel(&hidlModel); 249ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet const std::vector<std::shared_ptr<Device>>& devices = DeviceManager::get()->getDrivers(); 250ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet for (const auto& device : devices) { 251ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet hidl_vec<bool> supports; 2521f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross LOG(DEBUG) << "Checking " << device->getName(); 253ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet device->getSupportedOperations(hidlModel, &supports); 254ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet if (std::find(supports.begin(), supports.end(), false) == supports.end()) { 2551f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross LOG(DEBUG) << "ExecutionBuilder::startCompute (without plan) on " << device->getName(); 256b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross StepExecutor executor(this, mModel, device->getInterface(), 257b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross nullptr /* no IPreparedModel, so compile */); 258b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross executor.mapInputsAndOutputsTrivially(); 259033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return executor.startCompute(synchronizationCallback); 260ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet } 261ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet } 262ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet // If none can, run on the CPU. 2631f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross LOG(DEBUG) << "ExecutionBuilder::startCompute (without plan) on CPU"; 264b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross StepExecutor executor(this, mModel, 265b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross nullptr /* no IDevice, so CPU */, 266b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross nullptr /* no IPreparedModel */); 267b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross executor.mapInputsAndOutputsTrivially(); 268033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return executor.startCompute(synchronizationCallback); 269707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 270707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 271707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet// Figures out how to place each of the input or outputs in a buffer. This just does the layout, 2728b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet// it does not copy data. Aligns each input a bit. 273b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Grossint StepExecutor::allocatePointerArgumentsToPool(std::vector<ModelArgumentInfo>* args, 274b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross Memory* memory) { 2758b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t nextPoolIndex = mMemories.size(); 276707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet int64_t total = 0; 2778b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& info : *args) { 2788b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (info.state == ModelArgumentInfo::POINTER) { 2798b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet DataLocation& loc = info.locationAndDimension.location; 2808b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // TODO Good enough alignment? 2818b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet total += alignBytesNeeded(static_cast<uint32_t>(total), loc.length); 2828b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet loc.poolIndex = nextPoolIndex; 2838b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet loc.offset = static_cast<uint32_t>(total); 2848b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet total += loc.length; 2858b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 2868b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet }; 287707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (total > 0xFFFFFFFF) { 2883ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_startCompute Size of all inputs or outputs exceeds " 289707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet "2^32."; 290707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 291707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 2928b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet hidl_memory hidlMemory; 2938b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (total > 0) { 294389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet memory->create(total); // TODO check error 2958b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet mMemories.add(memory); 2968b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 297707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 298707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 299707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 3008b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouilletstatic void copyLocationAndDimension(const std::vector<ModelArgumentInfo>& argumentInfos, 301a4e2ee81d015b5e26897d18b18dd50044faba62fJean-Luc Brouillet hidl_vec<RequestArgument>* ioInfos) { 3028b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet size_t count = argumentInfos.size(); 3038b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet ioInfos->resize(count); 3048b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (size_t i = 0; i < count; i++) { 3058b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet (*ioInfos)[i] = argumentInfos[i].locationAndDimension; 3068b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 3078b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet} 3088b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet 309b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David GrossStepExecutor::StepExecutor(const ExecutionBuilder* executionBuilder, 310b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross const ModelBuilder* model, 311b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross sp<IDevice> driver, sp<IPreparedModel> preparedModel) : 312b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mExecutionBuilder(executionBuilder), mModel(model), 313891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross mDriver(driver), mPreparedModel(preparedModel), 314891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross mInputs(model->inputCount()), mOutputs(model->outputCount()) {} 315b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 316b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Grossvoid StepExecutor::mapInputsAndOutputsTrivially() { 317b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mInputs = mExecutionBuilder->mInputs; 318b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mOutputs = mExecutionBuilder->mOutputs; 319b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mMemories = mExecutionBuilder->mMemories; 320b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross} 321b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 322891b10f7048c62a37a74c4b570be220089dfd55eDavid Grossvoid StepExecutor::mapInputOrOutput(const ModelArgumentInfo& builderInputOrOutput, 323891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross ModelArgumentInfo* executorInputOrOutput) { 324891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross *executorInputOrOutput = builderInputOrOutput; 325891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross switch (executorInputOrOutput->state) { 326891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross default: 327891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross nnAssert(!"unexpected ModelArgumentInfo::state"); 328891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross case ModelArgumentInfo::POINTER: 329891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross case ModelArgumentInfo::UNSPECIFIED: 330891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross break; 331891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross case ModelArgumentInfo::MEMORY: { 332891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const uint32_t builderPoolIndex = 333891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross builderInputOrOutput.locationAndDimension.location.poolIndex; 334891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const Memory* memory = mExecutionBuilder->mMemories[builderPoolIndex]; 335891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const uint32_t executorPoolIndex = mMemories.add(memory); 336891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross executorInputOrOutput->locationAndDimension.location.poolIndex = 337891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross executorPoolIndex; 338891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross break; 339891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 340891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 341891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross} 342891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross 34396811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Grossint StepExecutor::setInputOrOutputFromTemporaryMemory(const Operand& inputOrOutputOperand, 34496811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross const Memory* memory, 34596811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross ModelArgumentInfo* inputOrOutputInfo) { 34696811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // Should be similar to 34796811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // ExecutionBuilder::setInputFromMemory() 34896811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // ExecutionBuilder::setOutputFromMemory() 34996811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 35096811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross uint32_t poolIndex = mMemories.add(memory); 35196811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross return inputOrOutputInfo->setFromTemporaryMemory(inputOrOutputOperand, poolIndex); 35296811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross} 35396811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 354033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint StepExecutor::startCompute(sp<ExecutionCallback>* synchronizationCallback) { 355b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross if (mDriver == nullptr) { 356033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return startComputeOnCpu(synchronizationCallback); 357b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross } else { 358033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return startComputeOnDevice(synchronizationCallback); 359b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross } 360b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross} 361b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 362033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint StepExecutor::startComputeOnDevice(sp<ExecutionCallback>* synchronizationCallback) { 363b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross nnAssert(mDriver != nullptr); 364b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 365033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = nullptr; 366425b2594c76e934dfdbc93209253e3c189571149David Gross 367b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross // TODO: Remove the mPreparedModel == nullptr case once we've fully integrated 3681f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // ExecutionPlan with the compilation and execution phases of the NN API 369b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross if (mPreparedModel == nullptr) { 3701f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross Model model; 3711f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mModel->setHidlModel(&model); 3721f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross 3731f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // TODO Dangerous! In async, the model will outlive it here. Safe for now 374033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback(); 375033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler Return<ErrorStatus> prepareLaunchStatus = 376033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler mDriver->prepareModel(model, preparedModelCallback); 377033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler if (!prepareLaunchStatus.isOk() || prepareLaunchStatus != ErrorStatus::NONE) { 378033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return ANEURALNETWORKS_OP_FAILED; 379033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler } 3805f916fc9a7ae95f172492bdfe5344c37beff3a6fMichael Butler 381033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // Immediately synchronize with callback object for now 3821f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // TODO: change to asynchronous later 383033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler preparedModelCallback->wait(); 384033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); 385033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler mPreparedModel = preparedModelCallback->getPreparedModel(); 386033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler if (prepareReturnStatus != ErrorStatus::NONE || mPreparedModel == nullptr) { 3871f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross return ANEURALNETWORKS_OP_FAILED; 3881f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross } 389707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 390707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 391e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // We separate the input & output pools so that we reduce the copying done if we 392e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // do an eventual remoting (hidl_memory->update()). We could also use it to set 393e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // protection on read only memory but that's not currently done. 394e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross Memory inputPointerArguments; 395e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross Memory outputPointerArguments; 396e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross 3978b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // Layout the input and output data 398e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross int n = allocatePointerArgumentsToPool(&mInputs, &inputPointerArguments); 399707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 400707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return n; 401707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 402e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross n = allocatePointerArgumentsToPool(&mOutputs, &outputPointerArguments); 403707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 404707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return n; 405707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 406707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 4078b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // Copy the input data that was specified via a pointer. 408e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // inputPointerArguments.update(); 4098b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& info : mInputs) { 4108b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (info.state == ModelArgumentInfo::POINTER) { 4118b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet DataLocation& loc = info.locationAndDimension.location; 4122150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet uint8_t* data = nullptr; 413e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross int n = inputPointerArguments.getPointer(&data); 4142150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 4152150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet return n; 4162150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet } 4178b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet memcpy(data + loc.offset, info.buffer, loc.length); 4188b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 419707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 420e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // TODO: Add inputPointerArguments.commit() and .update() at all the right places 421707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 422707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet Request request; 4238b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet copyLocationAndDimension(mInputs, &request.inputs); 4248b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet copyLocationAndDimension(mOutputs, &request.outputs); 4258b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = mMemories.size(); 4268b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet request.pools.resize(count); 4278b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (uint32_t i = 0; i < count; i++) { 4288b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet request.pools[i] = mMemories[i]->getHidlMemory(); 4298b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 430707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 431033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // Prepare the callback for asynchronous execution. sp<ExecutionCallback> 432033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // object is returned when the execution has been successfully launched, 433033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // otherwise a nullptr is returned. The executionCallback is abstracted in 434033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // the NN API as an "event". 435033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // 436033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // The sp is used for ref-counting purposes. Without it, the HIDL service 437033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // could attempt to communicate with a dead callback object. 438689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // 439033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // TODO: Explain the "dead callback" problem further, either here or 440689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // in the design document. 441033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler sp<ExecutionCallback> executionCallback = new ExecutionCallback(); 442689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 443b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross LOG(DEBUG) << "Before mPreparedModel->execute() " << toString(request); 4443ced3cfd5b8f22b632c35f24e585c4847383b195David Gross // Execute. 445033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // TODO: What happens to the Callback if the service dies abnormally 446033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // -- won't that keep the Callback live forever, because the service 447689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // never has the opportunity to bump the reference count down? Or 448689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // maybe the HIDL infrastructure handles this magically? At worst, 449033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // it seems like this is a small memory leak, if the Callback stays 450689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // alive forever. 451033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler if (mPreparedModel->execute(request, executionCallback) != ErrorStatus::NONE) { 452707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet LOG(DEBUG) << "**Execute failed**"; 453707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_OP_FAILED; 454707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 455707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 456689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: Remove this synchronization point when the block of code below is 457689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // removed. 458033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler executionCallback->wait(); 459033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler Return<ErrorStatus> executionStatus = executionCallback->getStatus(); 460033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler if (!executionStatus.isOk() || executionStatus != ErrorStatus::NONE) { 461689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler LOG(DEBUG) << "**Execute async failed**"; 462689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler return ANEURALNETWORKS_OP_FAILED; 463689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler } 464689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 465707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // Copy the output data from shared memory to the output buffers. 466689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: Move this block of code somewhere else. It should not be in the 467689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // startCompute function. 468689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: outputMemory->update(); outputMemory->commit() 4698b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& info : mOutputs) { 4708b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (info.state == ModelArgumentInfo::POINTER) { 4718b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet DataLocation& loc = info.locationAndDimension.location; 4722150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet uint8_t* data = nullptr; 473e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross int n = outputPointerArguments.getPointer(&data); 4742150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 4752150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet return n; 4762150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet } 4778b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet memcpy(info.buffer, data + loc.offset, loc.length); 4788b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 479707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 480b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross LOG(DEBUG) << "StepExecutor::startComputeOnDevice completed"; 481707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 482033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = executionCallback; 483707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 484707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 485707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 486689d892203c06c66c7bb2e374462a8434e40b75fMichael Butlerstatic void asyncStartComputeOnCpu(const Model& model, const Request& request, 487689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler const std::vector<RunTimePoolInfo>& runTimePoolInfos, 488033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler const sp<IExecutionCallback>& executionCallback) { 489689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler CpuExecutor executor; 490689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler int err = executor.run(model, request, runTimePoolInfos); 4915f916fc9a7ae95f172492bdfe5344c37beff3a6fMichael Butler ErrorStatus status = err == ANEURALNETWORKS_NO_ERROR ? 4925f916fc9a7ae95f172492bdfe5344c37beff3a6fMichael Butler ErrorStatus::NONE : ErrorStatus::GENERAL_FAILURE; 493033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler executionCallback->notify(status); 494689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler} 495689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 496033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint StepExecutor::startComputeOnCpu(sp<ExecutionCallback>* synchronizationCallback) { 497707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // TODO: use a thread pool 498689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 4991f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross Model model; 5001f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mModel->setHidlModel(&model); 5011f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross 502033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // Prepare the callback for asynchronous execution. sp<ExecutionCallback> 503033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // object is returned when the execution has been successfully launched, 504033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // otherwise a nullptr is returned. The executionCallback is abstracted in 505033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // the NN API as an "event". 506033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler sp<ExecutionCallback> executionCallback = new ExecutionCallback(); 507033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = nullptr; 508707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 509707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet std::vector<RunTimePoolInfo> runTimePoolInfos; 5108b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = mMemories.size(); 5118b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet runTimePoolInfos.resize(count); 5128b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (uint32_t i = 0; i < count; i++) { 5138b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet const Memory* mem = mMemories[i]; 5148b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet runTimePoolInfos[i].set(mem->getHidlMemory()); 5158b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 5168b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // Create as many pools as there are input / output. 5178b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet auto fixPointerArguments = [&runTimePoolInfos](std::vector<ModelArgumentInfo>& argumentInfos) { 5188b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (ModelArgumentInfo& argumentInfo : argumentInfos) { 5198b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (argumentInfo.state == ModelArgumentInfo::POINTER) { 520389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet RunTimePoolInfo runTimeInfo = { 521389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet .buffer = static_cast<uint8_t*>(argumentInfo.buffer)}; 5228b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet argumentInfo.locationAndDimension.location.poolIndex = 523389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet static_cast<uint32_t>(runTimePoolInfos.size()); 5248b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet argumentInfo.locationAndDimension.location.offset = 0; 5258b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet runTimePoolInfos.push_back(runTimeInfo); 5268b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 5278b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 5288b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet }; 5298b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet fixPointerArguments(mInputs); 5308b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet fixPointerArguments(mOutputs); 531707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 5328b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet Request request; 5338b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet copyLocationAndDimension(mInputs, &request.inputs); 5348b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet copyLocationAndDimension(mOutputs, &request.outputs); 535707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 536689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: should model be moved with a std::cref? 537689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler std::thread thread(asyncStartComputeOnCpu, model, std::move(request), 538033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler std::move(runTimePoolInfos), executionCallback); 539033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler executionCallback->bind_thread(std::move(thread)); 540689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 541033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = executionCallback; 542689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler return ANEURALNETWORKS_NO_ERROR; 543707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 544707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 545389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} // namespace nn 546389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} // namespace android 547