ExecutionBuilder.cpp revision e3178825b8686f3300a895572691a2e8c1f0676b
1707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet/* 2707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * Copyright (C) 2017 The Android Open Source Project 3707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * 4707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * Licensed under the Apache License, Version 2.0 (the "License"); 5707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * you may not use this file except in compliance with the License. 6707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * You may obtain a copy of the License at 7707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * 8707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * http://www.apache.org/licenses/LICENSE-2.0 9707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * 10707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * Unless required by applicable law or agreed to in writing, software 11707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * distributed under the License is distributed on an "AS IS" BASIS, 12707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * See the License for the specific language governing permissions and 14707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * limitations under the License. 15707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet */ 16707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 173ced3cfd5b8f22b632c35f24e585c4847383b195David Gross#define LOG_TAG "ExecutionBuilder" 18707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 193ced3cfd5b8f22b632c35f24e585c4847383b195David Gross#include "ExecutionBuilder.h" 20707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 2183e24dc4706a5b7089881a55daf05b3924fab3b7David Gross#include "CompilationBuilder.h" 22707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "CpuExecutor.h" 23707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "HalInterfaces.h" 24707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "Manager.h" 25707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "ModelBuilder.h" 26707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 27689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler#include <mutex> 28389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet#include <thread> 29689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler#include <vector> 30689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 31707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouilletnamespace android { 32707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouilletnamespace nn { 33707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 34389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouilletint ModelArgumentInfo::setFromPointer(const Operand& operand, 35389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet const ANeuralNetworksOperandType* type, void* data, 36389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t length) { 37389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet int n = updateDimensionInfo(operand, type); 38389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 39389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return n; 40389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 4162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet if (data == nullptr) { 4262cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet if (length) { 4362cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet LOG(ERROR) << "Setting argument as having no value but non-zero length passed."; 4462cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 4562cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet } 4662cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet state = ModelArgumentInfo::HAS_NO_VALUE; 4762cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet } else { 4862cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet state = ModelArgumentInfo::POINTER; 4962cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet } 50389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet buffer = data; 5162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet locationAndLength = {.poolIndex = 0, .offset = 0, .length = length}; 52389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 53389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} 54389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet 55389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouilletint ModelArgumentInfo::setFromMemory(const Operand& operand, const ANeuralNetworksOperandType* type, 56389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t poolIndex, uint32_t offset, uint32_t length) { 57389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet int n = updateDimensionInfo(operand, type); 58389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 59389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return n; 60389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 61389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet state = ModelArgumentInfo::MEMORY; 6262cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet locationAndLength = {.poolIndex = poolIndex, .offset = offset, .length = length}; 63389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet buffer = nullptr; 64389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 65389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} 66389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet 6796811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Grossint ModelArgumentInfo::setFromTemporaryMemory(const Operand& operand, uint32_t poolIndex) { 684d83c52f52613585f7b86368be762b2857f7460fDavid Gross dimensions = operand.dimensions; 6996811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross state = ModelArgumentInfo::MEMORY; 704d83c52f52613585f7b86368be762b2857f7460fDavid Gross locationAndLength = 7196811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross {.poolIndex = poolIndex, .offset = 0, .length = sizeOfData(operand)}; 7296811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross buffer = nullptr; 7396811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross return ANEURALNETWORKS_NO_ERROR; 7496811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross} 7596811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 76389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouilletint ModelArgumentInfo::updateDimensionInfo(const Operand& operand, 77389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet const ANeuralNetworksOperandType* newType) { 78389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (newType == nullptr) { 7962cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet dimensions = hidl_vec<uint32_t>(); 80389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } else { 81d2d0c031c43e8e5aafc75e8a652d79bcc2aaca99Jean-Luc Brouillet uint32_t count = newType->dimensionCount; 82389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (static_cast<OperandType>(newType->type) != operand.type || 83389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet count != operand.dimensions.size()) { 843ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setInput/Output incompatible types"; 85389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 86389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 87389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet for (uint32_t i = 0; i < count; i++) { 8862cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet dimensions[i] = newType->dimensions[i]; 89389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 90389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 91389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 92389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} 93389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet 943ced3cfd5b8f22b632c35f24e585c4847383b195David GrossExecutionBuilder::ExecutionBuilder(const CompilationBuilder* compilation) : 9583e24dc4706a5b7089881a55daf05b3924fab3b7David Gross mModel(compilation->mModel), 961f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mPlan(&compilation->mPlan), 9783e24dc4706a5b7089881a55daf05b3924fab3b7David Gross mInputs(mModel->inputCount()), 9883e24dc4706a5b7089881a55daf05b3924fab3b7David Gross mOutputs(mModel->outputCount()), 9983e24dc4706a5b7089881a55daf05b3924fab3b7David Gross mMemories(mModel->getMemories()) { 1003ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(DEBUG) << "ExecutionBuilder::ExecutionBuilder"; 101707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 102707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1033ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setInput(uint32_t index, const ANeuralNetworksOperandType* type, 104e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const void* buffer, size_t length) { 105707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mInputs.size()); 106707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (index >= count) { 1073ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setInput bad index " << index << " " << count; 108707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 109707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 110e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (type != nullptr) { 111e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet int n = validateOperandType(*type, "ANeuralNetworksExecution_setInput", false); 112e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 113e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return n; 114e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 115e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 116e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (length > 0xFFFFFFFF) { 117e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet LOG(ERROR) << "ANeuralNetworksExecution_setInput input exceeds max length " << length; 118e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 119e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 120e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet uint32_t l = static_cast<uint32_t>(length); 121389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return mInputs[index].setFromPointer(mModel->getInputOperand(index), type, 122e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const_cast<void*>(buffer), l); 123707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 124707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1253ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setInputFromMemory(uint32_t index, const ANeuralNetworksOperandType* type, 126e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const Memory* memory, size_t offset, size_t length) { 12796811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // Should be similar to StepExecutor::setInputOrOutputFromTemporaryMemory() 12896811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 1298b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mInputs.size()); 1308b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (index >= count) { 1313ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setInputFromMemory bad index " << index << " " 1328b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet << count; 1338b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 1348b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 135105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang if (!memory->validateSize(offset, length)) { 136105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang return ANEURALNETWORKS_BAD_DATA; 137105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang } 138e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet // TODO validate the rest 139389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t poolIndex = mMemories.add(memory); 140389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return mInputs[index].setFromMemory(mModel->getInputOperand(index), type, poolIndex, offset, 141389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet length); 142707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 143707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1443ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setOutput(uint32_t index, const ANeuralNetworksOperandType* type, void* buffer, 145e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet size_t length) { 146707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mOutputs.size()); 147707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (index >= count) { 1483ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setOutput bad index " << index << " " << count; 149707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 150707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 151e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (type != nullptr) { 152e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet int n = validateOperandType(*type, "ANeuralNetworksExecution_setOutput", false); 153e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 154e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return n; 155e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 156e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 157e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (length > 0xFFFFFFFF) { 158e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet LOG(ERROR) << "ANeuralNetworksExecution_setOutput input exceeds max length " << length; 159e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 160e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 161e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet uint32_t l = static_cast<uint32_t>(length); 162e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return mOutputs[index].setFromPointer(mModel->getOutputOperand(index), type, buffer, l); 163707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 164707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1653ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setOutputFromMemory(uint32_t index, const ANeuralNetworksOperandType* type, 166e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const Memory* memory, size_t offset, size_t length) { 16796811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // Should be similar to StepExecutor::setInputOrOutputFromTemporaryMemory() 16896811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 1698b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mOutputs.size()); 1708b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (index >= count) { 1713ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setOutputFromMemory bad index " << index << " " 1728b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet << count; 1738b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 1748b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 175105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang if (!memory->validateSize(offset, length)) { 176105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang return ANEURALNETWORKS_BAD_DATA; 177105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang } 178e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet // TODO validate the rest 179389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t poolIndex = mMemories.add(memory); 180389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return mOutputs[index].setFromMemory(mModel->getOutputOperand(index), type, poolIndex, offset, 181389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet length); 182707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 183707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 184e3178825b8686f3300a895572691a2e8c1f0676bDavid Grossstatic void asyncStartComputePartitioned(const ExecutionPlan* plan, 185e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross std::shared_ptr<ExecutionPlan::Controller> controller, 186e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross const sp<IExecutionCallback>& executionCallback) { 187e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross LOG(DEBUG) << "ExecutionBuilder::startCompute (from plan, iteratively)"; 188e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross while (true) { 189e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross std::shared_ptr<StepExecutor> executor; 190e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross LOG(DEBUG) << "looking for next StepExecutor"; 191e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross int n = plan->next(controller, &executor); 192e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross if (n != ANEURALNETWORKS_NO_ERROR || executor == nullptr) { 193e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross executionCallback->notify( 194e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross n == ANEURALNETWORKS_NO_ERROR ? ErrorStatus::NONE : ErrorStatus::GENERAL_FAILURE); 195e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross return; 196e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross } 197e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross 198e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross sp<ExecutionCallback> stepCallback; 199e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross n = executor->startCompute(&stepCallback); 200e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross if (n != ANEURALNETWORKS_NO_ERROR) { 201e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross executionCallback->notify(ErrorStatus::GENERAL_FAILURE); 202e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross return; 203e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross } 204e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross stepCallback->wait(); 205e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross ErrorStatus status = stepCallback->getStatus(); 206e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross if (status != ErrorStatus::NONE) { 207e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross executionCallback->notify(status); 208e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross return; 209e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross } 210e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross } 211e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross} 212e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross 213033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint ExecutionBuilder::startCompute(sp<ExecutionCallback>* synchronizationCallback) { 214033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = nullptr; 215425b2594c76e934dfdbc93209253e3c189571149David Gross 216707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // TODO validate that we have full types for all inputs and outputs, 217707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // that the graph is not cyclic, 218f1817c663af4f22bc089ef82cd50df4186422c42Yang Ni 2198b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& p : mInputs) { 220389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (p.state == ModelArgumentInfo::UNSPECIFIED) { 2213ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_startCompute not all inputs specified"; 222707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 223707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 224707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 2258b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& p : mOutputs) { 226389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (p.state == ModelArgumentInfo::UNSPECIFIED) { 2273ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_startCompute not all outputs specified"; 228707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 229707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 230707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 231707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 2321f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // TODO: Remove the non-plan-based path once we've fully integrated ExecutionPlan 233a2a03635c8f215cb75be68ff1939bf4dec285ef8David Gross // with the compilation and execution phases of the NN API? Or retain that path 234a2a03635c8f215cb75be68ff1939bf4dec285ef8David Gross // as a fallback in the case of partitioning failure? 235b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross // 236b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross // TODO: Entire plan-based-path should run in an asynchronous thread -- 237b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross // take the asynchronous thread logic out of startComputeOnCpu() and use 238b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross // it to wrap the plan-based-path. 239e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross const uint32_t partitioning = DeviceManager::get()->getPartitioning(); 240a2a03635c8f215cb75be68ff1939bf4dec285ef8David Gross if (partitioning > 0) { 241a2a03635c8f215cb75be68ff1939bf4dec285ef8David Gross std::shared_ptr<ExecutionPlan::Controller> controller = mPlan->makeController(this); 242a2a03635c8f215cb75be68ff1939bf4dec285ef8David Gross if (controller == nullptr) { 243e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross if (!DeviceManager::partitioningAllowsFallback(partitioning)) { 244a2a03635c8f215cb75be68ff1939bf4dec285ef8David Gross return ANEURALNETWORKS_OP_FAILED; 245a2a03635c8f215cb75be68ff1939bf4dec285ef8David Gross } 246a2a03635c8f215cb75be68ff1939bf4dec285ef8David Gross } else { 247e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross // TODO: use a thread pool 248b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 249e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross // Prepare the callback for asynchronous execution. 250e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross // sp<ExecutionCallback> object is returned when the 251e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross // execution has been successfully launched, otherwise a 252e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross // nullptr is returned. The executionCallback is 253e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross // abstracted in the NN API as an "event". 254e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross sp<ExecutionCallback> executionCallback = new ExecutionCallback(); 255e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross std::thread thread(asyncStartComputePartitioned, mPlan, controller, executionCallback); 256e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross executionCallback->bind_thread(std::move(thread)); 257e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross *synchronizationCallback = executionCallback; 258e3178825b8686f3300a895572691a2e8c1f0676bDavid Gross return ANEURALNETWORKS_NO_ERROR; 2591f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross } 2601f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross } 261ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet 262ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet // Find a driver that can handle all the operations. 2631f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross Model hidlModel; 2641f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mModel->setHidlModel(&hidlModel); 265ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet const std::vector<std::shared_ptr<Device>>& devices = DeviceManager::get()->getDrivers(); 266ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet for (const auto& device : devices) { 267ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet hidl_vec<bool> supports; 2681f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross LOG(DEBUG) << "Checking " << device->getName(); 269ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet device->getSupportedOperations(hidlModel, &supports); 270ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet if (std::find(supports.begin(), supports.end(), false) == supports.end()) { 2711f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross LOG(DEBUG) << "ExecutionBuilder::startCompute (without plan) on " << device->getName(); 272b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross StepExecutor executor(this, mModel, device->getInterface(), 273b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross nullptr /* no IPreparedModel, so compile */); 274b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross executor.mapInputsAndOutputsTrivially(); 275033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return executor.startCompute(synchronizationCallback); 276ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet } 277ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet } 278ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet // If none can, run on the CPU. 2791f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross LOG(DEBUG) << "ExecutionBuilder::startCompute (without plan) on CPU"; 280b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross StepExecutor executor(this, mModel, 281b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross nullptr /* no IDevice, so CPU */, 282b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross nullptr /* no IPreparedModel */); 283b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross executor.mapInputsAndOutputsTrivially(); 284033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return executor.startCompute(synchronizationCallback); 285707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 286707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 287707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet// Figures out how to place each of the input or outputs in a buffer. This just does the layout, 2888b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet// it does not copy data. Aligns each input a bit. 289b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Grossint StepExecutor::allocatePointerArgumentsToPool(std::vector<ModelArgumentInfo>* args, 290b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross Memory* memory) { 2918b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t nextPoolIndex = mMemories.size(); 292707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet int64_t total = 0; 2938b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& info : *args) { 2948b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (info.state == ModelArgumentInfo::POINTER) { 29562cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet DataLocation& loc = info.locationAndLength; 2968b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // TODO Good enough alignment? 2978b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet total += alignBytesNeeded(static_cast<uint32_t>(total), loc.length); 2988b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet loc.poolIndex = nextPoolIndex; 2998b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet loc.offset = static_cast<uint32_t>(total); 3008b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet total += loc.length; 3018b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 3028b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet }; 303707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (total > 0xFFFFFFFF) { 3043ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_startCompute Size of all inputs or outputs exceeds " 305707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet "2^32."; 306707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 307707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 3088b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet hidl_memory hidlMemory; 3098b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (total > 0) { 310389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet memory->create(total); // TODO check error 3118b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet mMemories.add(memory); 3128b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 313707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 314707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 315707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 31662cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouilletstatic void setRequestArgumentArray(const std::vector<ModelArgumentInfo>& argumentInfos, 317a4e2ee81d015b5e26897d18b18dd50044faba62fJean-Luc Brouillet hidl_vec<RequestArgument>* ioInfos) { 3188b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet size_t count = argumentInfos.size(); 3198b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet ioInfos->resize(count); 3208b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (size_t i = 0; i < count; i++) { 32162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet const auto& info = argumentInfos[i]; 32262cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet (*ioInfos)[i] = { .hasNoValue = info.state == ModelArgumentInfo::HAS_NO_VALUE, 32362cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet .location = info.locationAndLength, 32462cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet .dimensions = info.dimensions, 32562cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet }; 3268b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 3278b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet} 3288b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet 329b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David GrossStepExecutor::StepExecutor(const ExecutionBuilder* executionBuilder, 330b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross const ModelBuilder* model, 331b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross sp<IDevice> driver, sp<IPreparedModel> preparedModel) : 332b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mExecutionBuilder(executionBuilder), mModel(model), 333891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross mDriver(driver), mPreparedModel(preparedModel), 334891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross mInputs(model->inputCount()), mOutputs(model->outputCount()) {} 335b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 336b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Grossvoid StepExecutor::mapInputsAndOutputsTrivially() { 337b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mInputs = mExecutionBuilder->mInputs; 338b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mOutputs = mExecutionBuilder->mOutputs; 339b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mMemories = mExecutionBuilder->mMemories; 340b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross} 341b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 342891b10f7048c62a37a74c4b570be220089dfd55eDavid Grossvoid StepExecutor::mapInputOrOutput(const ModelArgumentInfo& builderInputOrOutput, 343891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross ModelArgumentInfo* executorInputOrOutput) { 344891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross *executorInputOrOutput = builderInputOrOutput; 345891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross switch (executorInputOrOutput->state) { 346891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross default: 347891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross nnAssert(!"unexpected ModelArgumentInfo::state"); 348891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross case ModelArgumentInfo::POINTER: 349891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross case ModelArgumentInfo::UNSPECIFIED: 350891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross break; 351891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross case ModelArgumentInfo::MEMORY: { 352891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const uint32_t builderPoolIndex = 35362cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet builderInputOrOutput.locationAndLength.poolIndex; 354891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const Memory* memory = mExecutionBuilder->mMemories[builderPoolIndex]; 355891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const uint32_t executorPoolIndex = mMemories.add(memory); 35662cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet executorInputOrOutput->locationAndLength.poolIndex = 357891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross executorPoolIndex; 358891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross break; 359891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 360891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 361891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross} 362891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross 36396811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Grossint StepExecutor::setInputOrOutputFromTemporaryMemory(const Operand& inputOrOutputOperand, 36496811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross const Memory* memory, 36596811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross ModelArgumentInfo* inputOrOutputInfo) { 36696811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // Should be similar to 36796811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // ExecutionBuilder::setInputFromMemory() 36896811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross // ExecutionBuilder::setOutputFromMemory() 36996811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 37096811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross uint32_t poolIndex = mMemories.add(memory); 37196811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross return inputOrOutputInfo->setFromTemporaryMemory(inputOrOutputOperand, poolIndex); 37296811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross} 37396811e2b1347889a25bd9686f47ca3cbf061fb1bDavid Gross 374033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint StepExecutor::startCompute(sp<ExecutionCallback>* synchronizationCallback) { 375b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross if (mDriver == nullptr) { 376033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return startComputeOnCpu(synchronizationCallback); 377b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross } else { 378033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return startComputeOnDevice(synchronizationCallback); 379b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross } 380b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross} 381b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 382033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint StepExecutor::startComputeOnDevice(sp<ExecutionCallback>* synchronizationCallback) { 383b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross nnAssert(mDriver != nullptr); 384b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 385033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = nullptr; 386425b2594c76e934dfdbc93209253e3c189571149David Gross 387b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross // TODO: Remove the mPreparedModel == nullptr case once we've fully integrated 3881f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // ExecutionPlan with the compilation and execution phases of the NN API 389b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross if (mPreparedModel == nullptr) { 3901f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross Model model; 3911f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mModel->setHidlModel(&model); 3921f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross 3931f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // TODO Dangerous! In async, the model will outlive it here. Safe for now 394033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback(); 395033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler Return<ErrorStatus> prepareLaunchStatus = 396033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler mDriver->prepareModel(model, preparedModelCallback); 397033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler if (!prepareLaunchStatus.isOk() || prepareLaunchStatus != ErrorStatus::NONE) { 398033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler return ANEURALNETWORKS_OP_FAILED; 399033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler } 4005f916fc9a7ae95f172492bdfe5344c37beff3a6fMichael Butler 401033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // Immediately synchronize with callback object for now 4021f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // TODO: change to asynchronous later 403033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler preparedModelCallback->wait(); 404033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); 405033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler mPreparedModel = preparedModelCallback->getPreparedModel(); 406033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler if (prepareReturnStatus != ErrorStatus::NONE || mPreparedModel == nullptr) { 4071f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross return ANEURALNETWORKS_OP_FAILED; 4081f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross } 409707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 410707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 411e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // We separate the input & output pools so that we reduce the copying done if we 412e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // do an eventual remoting (hidl_memory->update()). We could also use it to set 413e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // protection on read only memory but that's not currently done. 414e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross Memory inputPointerArguments; 415e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross Memory outputPointerArguments; 416e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross 4178b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // Layout the input and output data 418e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross int n = allocatePointerArgumentsToPool(&mInputs, &inputPointerArguments); 419707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 420707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return n; 421707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 422e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross n = allocatePointerArgumentsToPool(&mOutputs, &outputPointerArguments); 423707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 424707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return n; 425707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 426707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 4278b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // Copy the input data that was specified via a pointer. 428e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // inputPointerArguments.update(); 4298b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& info : mInputs) { 4308b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (info.state == ModelArgumentInfo::POINTER) { 43162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet DataLocation& loc = info.locationAndLength; 4322150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet uint8_t* data = nullptr; 433e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross int n = inputPointerArguments.getPointer(&data); 4342150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 4352150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet return n; 4362150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet } 4378b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet memcpy(data + loc.offset, info.buffer, loc.length); 4388b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 439707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 440e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // TODO: Add inputPointerArguments.commit() and .update() at all the right places 441707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 442707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet Request request; 44362cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mInputs, &request.inputs); 44462cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mOutputs, &request.outputs); 4458b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = mMemories.size(); 4468b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet request.pools.resize(count); 4478b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (uint32_t i = 0; i < count; i++) { 4488b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet request.pools[i] = mMemories[i]->getHidlMemory(); 4498b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 450707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 451033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // Prepare the callback for asynchronous execution. sp<ExecutionCallback> 452033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // object is returned when the execution has been successfully launched, 453033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // otherwise a nullptr is returned. The executionCallback is abstracted in 454033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // the NN API as an "event". 455033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // 456033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // The sp is used for ref-counting purposes. Without it, the HIDL service 457033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // could attempt to communicate with a dead callback object. 458689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // 459033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // TODO: Explain the "dead callback" problem further, either here or 460689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // in the design document. 461033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler sp<ExecutionCallback> executionCallback = new ExecutionCallback(); 462689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 463b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross LOG(DEBUG) << "Before mPreparedModel->execute() " << toString(request); 4643ced3cfd5b8f22b632c35f24e585c4847383b195David Gross // Execute. 465033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // TODO: What happens to the Callback if the service dies abnormally 466033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // -- won't that keep the Callback live forever, because the service 467689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // never has the opportunity to bump the reference count down? Or 468689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // maybe the HIDL infrastructure handles this magically? At worst, 469033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // it seems like this is a small memory leak, if the Callback stays 470689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // alive forever. 471033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler if (mPreparedModel->execute(request, executionCallback) != ErrorStatus::NONE) { 472707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet LOG(DEBUG) << "**Execute failed**"; 473707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_OP_FAILED; 474707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 475707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 476689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: Remove this synchronization point when the block of code below is 477689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // removed. 478033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler executionCallback->wait(); 479033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler Return<ErrorStatus> executionStatus = executionCallback->getStatus(); 480033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler if (!executionStatus.isOk() || executionStatus != ErrorStatus::NONE) { 481689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler LOG(DEBUG) << "**Execute async failed**"; 482689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler return ANEURALNETWORKS_OP_FAILED; 483689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler } 484689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 485707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // Copy the output data from shared memory to the output buffers. 486689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: Move this block of code somewhere else. It should not be in the 487689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // startCompute function. 488689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: outputMemory->update(); outputMemory->commit() 4898b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& info : mOutputs) { 4908b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (info.state == ModelArgumentInfo::POINTER) { 49162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet DataLocation& loc = info.locationAndLength; 4922150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet uint8_t* data = nullptr; 493e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross int n = outputPointerArguments.getPointer(&data); 4942150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 4952150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet return n; 4962150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet } 4978b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet memcpy(info.buffer, data + loc.offset, loc.length); 4988b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 499707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 500b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross LOG(DEBUG) << "StepExecutor::startComputeOnDevice completed"; 501707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 502033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = executionCallback; 503707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 504707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 505707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 506689d892203c06c66c7bb2e374462a8434e40b75fMichael Butlerstatic void asyncStartComputeOnCpu(const Model& model, const Request& request, 507689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler const std::vector<RunTimePoolInfo>& runTimePoolInfos, 508033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler const sp<IExecutionCallback>& executionCallback) { 509689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler CpuExecutor executor; 510689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler int err = executor.run(model, request, runTimePoolInfos); 5115f916fc9a7ae95f172492bdfe5344c37beff3a6fMichael Butler ErrorStatus status = err == ANEURALNETWORKS_NO_ERROR ? 5125f916fc9a7ae95f172492bdfe5344c37beff3a6fMichael Butler ErrorStatus::NONE : ErrorStatus::GENERAL_FAILURE; 513033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler executionCallback->notify(status); 514689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler} 515689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 516033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butlerint StepExecutor::startComputeOnCpu(sp<ExecutionCallback>* synchronizationCallback) { 517707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // TODO: use a thread pool 518689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 5191f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross Model model; 5201f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mModel->setHidlModel(&model); 5211f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross 522033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // Prepare the callback for asynchronous execution. sp<ExecutionCallback> 523033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // object is returned when the execution has been successfully launched, 524033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // otherwise a nullptr is returned. The executionCallback is abstracted in 525033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler // the NN API as an "event". 526033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler sp<ExecutionCallback> executionCallback = new ExecutionCallback(); 527033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = nullptr; 528707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 529707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet std::vector<RunTimePoolInfo> runTimePoolInfos; 5308b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = mMemories.size(); 5318b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet runTimePoolInfos.resize(count); 5328b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (uint32_t i = 0; i < count; i++) { 5338b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet const Memory* mem = mMemories[i]; 5348b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet runTimePoolInfos[i].set(mem->getHidlMemory()); 5358b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 5368b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // Create as many pools as there are input / output. 5378b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet auto fixPointerArguments = [&runTimePoolInfos](std::vector<ModelArgumentInfo>& argumentInfos) { 5388b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (ModelArgumentInfo& argumentInfo : argumentInfos) { 5398b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (argumentInfo.state == ModelArgumentInfo::POINTER) { 540389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet RunTimePoolInfo runTimeInfo = { 541389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet .buffer = static_cast<uint8_t*>(argumentInfo.buffer)}; 54262cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet argumentInfo.locationAndLength.poolIndex = 543389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet static_cast<uint32_t>(runTimePoolInfos.size()); 54462cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet argumentInfo.locationAndLength.offset = 0; 5458b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet runTimePoolInfos.push_back(runTimeInfo); 5468b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 5478b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 5488b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet }; 5498b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet fixPointerArguments(mInputs); 5508b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet fixPointerArguments(mOutputs); 551707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 5528b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet Request request; 55362cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mInputs, &request.inputs); 55462cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mOutputs, &request.outputs); 555707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 556689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: should model be moved with a std::cref? 557689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler std::thread thread(asyncStartComputeOnCpu, model, std::move(request), 558033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler std::move(runTimePoolInfos), executionCallback); 559033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler executionCallback->bind_thread(std::move(thread)); 560689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 561033b8a6ce8ebd2a01ecccc6bae96d0fff8d4964eMichael Butler *synchronizationCallback = executionCallback; 562689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler return ANEURALNETWORKS_NO_ERROR; 563707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 564707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 565389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} // namespace nn 566389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} // namespace android 567