ExecutionBuilder.cpp revision 62cc2758c1c2d303861e209f26bddcf4d7564b73
1707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet/* 2707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * Copyright (C) 2017 The Android Open Source Project 3707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * 4707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * Licensed under the Apache License, Version 2.0 (the "License"); 5707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * you may not use this file except in compliance with the License. 6707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * You may obtain a copy of the License at 7707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * 8707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * http://www.apache.org/licenses/LICENSE-2.0 9707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * 10707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * Unless required by applicable law or agreed to in writing, software 11707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * distributed under the License is distributed on an "AS IS" BASIS, 12707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * See the License for the specific language governing permissions and 14707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet * limitations under the License. 15707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet */ 16707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 173ced3cfd5b8f22b632c35f24e585c4847383b195David Gross#define LOG_TAG "ExecutionBuilder" 18707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 193ced3cfd5b8f22b632c35f24e585c4847383b195David Gross#include "ExecutionBuilder.h" 20707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 2183e24dc4706a5b7089881a55daf05b3924fab3b7David Gross#include "CompilationBuilder.h" 22707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "CpuExecutor.h" 23707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "HalInterfaces.h" 24707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "Manager.h" 25707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet#include "ModelBuilder.h" 26707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 27689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler#include <mutex> 28389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet#include <thread> 29689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler#include <vector> 30689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 31707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouilletnamespace android { 32707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouilletnamespace nn { 33707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 34389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouilletint ModelArgumentInfo::setFromPointer(const Operand& operand, 35389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet const ANeuralNetworksOperandType* type, void* data, 36389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t length) { 37389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet int n = updateDimensionInfo(operand, type); 38389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 39389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return n; 40389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 4162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet if (data == nullptr) { 4262cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet if (length) { 4362cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet LOG(ERROR) << "Setting argument as having no value but non-zero length passed."; 4462cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 4562cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet } 4662cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet state = ModelArgumentInfo::HAS_NO_VALUE; 4762cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet } else { 4862cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet state = ModelArgumentInfo::POINTER; 4962cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet } 50389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet buffer = data; 5162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet locationAndLength = {.poolIndex = 0, .offset = 0, .length = length}; 52389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 53389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} 54389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet 55389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouilletint ModelArgumentInfo::setFromMemory(const Operand& operand, const ANeuralNetworksOperandType* type, 56389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t poolIndex, uint32_t offset, uint32_t length) { 57389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet int n = updateDimensionInfo(operand, type); 58389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 59389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return n; 60389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 61389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet state = ModelArgumentInfo::MEMORY; 6262cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet locationAndLength = {.poolIndex = poolIndex, .offset = offset, .length = length}; 63389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet buffer = nullptr; 64389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 65389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} 66389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet 67389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouilletint ModelArgumentInfo::updateDimensionInfo(const Operand& operand, 68389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet const ANeuralNetworksOperandType* newType) { 69389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (newType == nullptr) { 7062cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet dimensions = hidl_vec<uint32_t>(); 71389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } else { 72d2d0c031c43e8e5aafc75e8a652d79bcc2aaca99Jean-Luc Brouillet uint32_t count = newType->dimensionCount; 73389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (static_cast<OperandType>(newType->type) != operand.type || 74389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet count != operand.dimensions.size()) { 753ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setInput/Output incompatible types"; 76389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 77389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 78389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet for (uint32_t i = 0; i < count; i++) { 7962cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet dimensions[i] = newType->dimensions[i]; 80389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 81389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet } 82389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 83389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} 84389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet 853ced3cfd5b8f22b632c35f24e585c4847383b195David GrossExecutionBuilder::ExecutionBuilder(const CompilationBuilder* compilation) : 8683e24dc4706a5b7089881a55daf05b3924fab3b7David Gross mModel(compilation->mModel), 871f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mPlan(&compilation->mPlan), 8883e24dc4706a5b7089881a55daf05b3924fab3b7David Gross mInputs(mModel->inputCount()), 8983e24dc4706a5b7089881a55daf05b3924fab3b7David Gross mOutputs(mModel->outputCount()), 9083e24dc4706a5b7089881a55daf05b3924fab3b7David Gross mMemories(mModel->getMemories()) { 913ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(DEBUG) << "ExecutionBuilder::ExecutionBuilder"; 92707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 93707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 943ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setInput(uint32_t index, const ANeuralNetworksOperandType* type, 95e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const void* buffer, size_t length) { 96707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mInputs.size()); 97707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (index >= count) { 983ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setInput bad index " << index << " " << count; 99707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 100707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 101e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (type != nullptr) { 102e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet int n = validateOperandType(*type, "ANeuralNetworksExecution_setInput", false); 103e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 104e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return n; 105e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 106e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 107e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (length > 0xFFFFFFFF) { 108e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet LOG(ERROR) << "ANeuralNetworksExecution_setInput input exceeds max length " << length; 109e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 110e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 111e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet uint32_t l = static_cast<uint32_t>(length); 112389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return mInputs[index].setFromPointer(mModel->getInputOperand(index), type, 113e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const_cast<void*>(buffer), l); 114707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 115707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1163ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setInputFromMemory(uint32_t index, const ANeuralNetworksOperandType* type, 117e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const Memory* memory, size_t offset, size_t length) { 1188b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mInputs.size()); 1198b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (index >= count) { 1203ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setInputFromMemory bad index " << index << " " 1218b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet << count; 1228b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 1238b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 124105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang if (!memory->validateSize(offset, length)) { 125105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang return ANEURALNETWORKS_BAD_DATA; 126105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang } 127e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet // TODO validate the rest 128389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t poolIndex = mMemories.add(memory); 129389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return mInputs[index].setFromMemory(mModel->getInputOperand(index), type, poolIndex, offset, 130389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet length); 131707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 132707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1333ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setOutput(uint32_t index, const ANeuralNetworksOperandType* type, void* buffer, 134e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet size_t length) { 135707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mOutputs.size()); 136707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (index >= count) { 1373ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setOutput bad index " << index << " " << count; 138707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 139707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 140e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (type != nullptr) { 141e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet int n = validateOperandType(*type, "ANeuralNetworksExecution_setOutput", false); 142e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 143e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return n; 144e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 145e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 146e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet if (length > 0xFFFFFFFF) { 147e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet LOG(ERROR) << "ANeuralNetworksExecution_setOutput input exceeds max length " << length; 148e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 149e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet } 150e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet uint32_t l = static_cast<uint32_t>(length); 151e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet return mOutputs[index].setFromPointer(mModel->getOutputOperand(index), type, buffer, l); 152707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 153707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1543ced3cfd5b8f22b632c35f24e585c4847383b195David Grossint ExecutionBuilder::setOutputFromMemory(uint32_t index, const ANeuralNetworksOperandType* type, 155e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet const Memory* memory, size_t offset, size_t length) { 1568b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = static_cast<uint32_t>(mOutputs.size()); 1578b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (index >= count) { 1583ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_setOutputFromMemory bad index " << index << " " 1598b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet << count; 1608b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 1618b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 162105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang if (!memory->validateSize(offset, length)) { 163105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang return ANEURALNETWORKS_BAD_DATA; 164105807d963d969197fe78185ed588bfad3dc0ea5Miao Wang } 165e127e49e67b53c96eb79f6a9c58f956ad4761227Jean-Luc Brouillet // TODO validate the rest 166389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet uint32_t poolIndex = mMemories.add(memory); 167389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet return mOutputs[index].setFromMemory(mModel->getOutputOperand(index), type, poolIndex, offset, 168389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet length); 169707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 170707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 171425b2594c76e934dfdbc93209253e3c189571149David Grossint ExecutionBuilder::startCompute(sp<Event>* event) { 172425b2594c76e934dfdbc93209253e3c189571149David Gross *event = nullptr; 173425b2594c76e934dfdbc93209253e3c189571149David Gross 174707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // TODO validate that we have full types for all inputs and outputs, 175707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // that the graph is not cyclic, 176f1817c663af4f22bc089ef82cd50df4186422c42Yang Ni 1778b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& p : mInputs) { 178389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (p.state == ModelArgumentInfo::UNSPECIFIED) { 1793ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_startCompute not all inputs specified"; 180707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 181707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 182707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 1838b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& p : mOutputs) { 184389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet if (p.state == ModelArgumentInfo::UNSPECIFIED) { 1853ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_startCompute not all outputs specified"; 186707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 187707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 188707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 189707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 1901f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // TODO: Remove the non-plan-based path once we've fully integrated ExecutionPlan 1911f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // with the compilation and execution phases of the NN API. 192b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross // 193b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross // TODO: Entire plan-based-path should run in an asynchronous thread -- 194b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross // take the asynchronous thread logic out of startComputeOnCpu() and use 195b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross // it to wrap the plan-based-path. 1961f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross#if NN_DEBUGGABLE 197891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross { 198891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const int partitioning = DeviceManager::get()->getPartitioning(); 199891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross if (partitioning > 0) { 200891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const bool simulation = !((partitioning > 1) && mPlan->shouldBeExecutable()); 201891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross LOG(DEBUG) << "ExecutionBuilder::startCompute" 202891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross << (simulation ? " SIMULATION" : "") 203891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross << " (from plan, iteratively)"; 204891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross ExecutionPlan::Controller controller = mPlan->makeController(this); 205891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross while (true) { 206891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross LOG(DEBUG) << "looking for next StepExecutor"; 207891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross std::shared_ptr<StepExecutor> executor; 208891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross int n = mPlan->next(&controller, &executor); 209891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross if (n != ANEURALNETWORKS_NO_ERROR || executor == nullptr) { 210891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross if (!simulation) { 211891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross return n; 212891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 213b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 214891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross // simulation 215891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross if (n != ANEURALNETWORKS_NO_ERROR) { 216891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross LOG(DEBUG) << "ExecutionBuilder::startCompute SIMULATION failed " 217891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross << "with error " << n; 218891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 219891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross break; 220891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 221891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross if (simulation) { 222891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross continue; 223891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 224891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross 225891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross n = executor->startCompute(event); 226891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross if (n != ANEURALNETWORKS_NO_ERROR) { 227891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross return n; 228891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 229891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross if ((*event)->wait() != Event::Status::SUCCESS) { 230891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross return ANEURALNETWORKS_OP_FAILED; 231891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 232b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross } 2331f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross } 2341f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross } 2351f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross#endif // NN_DEBUGGABLE 236ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet 237ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet // Find a driver that can handle all the operations. 2381f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross Model hidlModel; 2391f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mModel->setHidlModel(&hidlModel); 240ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet const std::vector<std::shared_ptr<Device>>& devices = DeviceManager::get()->getDrivers(); 241ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet for (const auto& device : devices) { 242ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet hidl_vec<bool> supports; 2431f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross LOG(DEBUG) << "Checking " << device->getName(); 244ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet device->getSupportedOperations(hidlModel, &supports); 245ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet if (std::find(supports.begin(), supports.end(), false) == supports.end()) { 2461f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross LOG(DEBUG) << "ExecutionBuilder::startCompute (without plan) on " << device->getName(); 247b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross StepExecutor executor(this, mModel, device->getInterface(), 248b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross nullptr /* no IPreparedModel, so compile */); 249b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross executor.mapInputsAndOutputsTrivially(); 250b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross return executor.startCompute(event); 251ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet } 252ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet } 253ef22aa5727b96e9a0863ef71cfbe3dbdac339408Jean-Luc Brouillet // If none can, run on the CPU. 2541f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross LOG(DEBUG) << "ExecutionBuilder::startCompute (without plan) on CPU"; 255b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross StepExecutor executor(this, mModel, 256b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross nullptr /* no IDevice, so CPU */, 257b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross nullptr /* no IPreparedModel */); 258b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross executor.mapInputsAndOutputsTrivially(); 259b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross return executor.startCompute(event); 260707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 261707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 262707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet// Figures out how to place each of the input or outputs in a buffer. This just does the layout, 2638b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet// it does not copy data. Aligns each input a bit. 264b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Grossint StepExecutor::allocatePointerArgumentsToPool(std::vector<ModelArgumentInfo>* args, 265b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross Memory* memory) { 2668b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t nextPoolIndex = mMemories.size(); 267707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet int64_t total = 0; 2688b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& info : *args) { 2698b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (info.state == ModelArgumentInfo::POINTER) { 27062cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet DataLocation& loc = info.locationAndLength; 2718b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // TODO Good enough alignment? 2728b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet total += alignBytesNeeded(static_cast<uint32_t>(total), loc.length); 2738b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet loc.poolIndex = nextPoolIndex; 2748b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet loc.offset = static_cast<uint32_t>(total); 2758b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet total += loc.length; 2768b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 2778b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet }; 278707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (total > 0xFFFFFFFF) { 2793ced3cfd5b8f22b632c35f24e585c4847383b195David Gross LOG(ERROR) << "ANeuralNetworksExecution_startCompute Size of all inputs or outputs exceeds " 280707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet "2^32."; 281707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_BAD_DATA; 282707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 2838b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet hidl_memory hidlMemory; 2848b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (total > 0) { 285389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet memory->create(total); // TODO check error 2868b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet mMemories.add(memory); 2878b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 288707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 289707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 290707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 29162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouilletstatic void setRequestArgumentArray(const std::vector<ModelArgumentInfo>& argumentInfos, 292a4e2ee81d015b5e26897d18b18dd50044faba62fJean-Luc Brouillet hidl_vec<RequestArgument>* ioInfos) { 2938b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet size_t count = argumentInfos.size(); 2948b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet ioInfos->resize(count); 2958b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (size_t i = 0; i < count; i++) { 29662cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet const auto& info = argumentInfos[i]; 29762cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet (*ioInfos)[i] = { .hasNoValue = info.state == ModelArgumentInfo::HAS_NO_VALUE, 29862cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet .location = info.locationAndLength, 29962cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet .dimensions = info.dimensions, 30062cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet }; 3018b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 3028b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet} 3038b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet 304b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David GrossStepExecutor::StepExecutor(const ExecutionBuilder* executionBuilder, 305b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross const ModelBuilder* model, 306b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross sp<IDevice> driver, sp<IPreparedModel> preparedModel) : 307b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mExecutionBuilder(executionBuilder), mModel(model), 308891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross mDriver(driver), mPreparedModel(preparedModel), 309891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross mInputs(model->inputCount()), mOutputs(model->outputCount()) {} 310b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 311b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Grossvoid StepExecutor::mapInputsAndOutputsTrivially() { 312b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mInputs = mExecutionBuilder->mInputs; 313b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mOutputs = mExecutionBuilder->mOutputs; 314b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mMemories = mExecutionBuilder->mMemories; 315b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross} 316b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 317891b10f7048c62a37a74c4b570be220089dfd55eDavid Grossvoid StepExecutor::mapInputOrOutput(const ModelArgumentInfo& builderInputOrOutput, 318891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross ModelArgumentInfo* executorInputOrOutput) { 319891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross *executorInputOrOutput = builderInputOrOutput; 320891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross switch (executorInputOrOutput->state) { 321891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross default: 322891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross nnAssert(!"unexpected ModelArgumentInfo::state"); 323891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross case ModelArgumentInfo::POINTER: 324891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross case ModelArgumentInfo::UNSPECIFIED: 325891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross break; 326891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross case ModelArgumentInfo::MEMORY: { 327891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const uint32_t builderPoolIndex = 32862cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet builderInputOrOutput.locationAndLength.poolIndex; 329891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const Memory* memory = mExecutionBuilder->mMemories[builderPoolIndex]; 330891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross const uint32_t executorPoolIndex = mMemories.add(memory); 33162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet executorInputOrOutput->locationAndLength.poolIndex = 332891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross executorPoolIndex; 333891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross break; 334891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 335891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross } 336891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross} 337891b10f7048c62a37a74c4b570be220089dfd55eDavid Gross 338b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Grossint StepExecutor::startCompute(sp<Event>* event) { 339b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross if (mDriver == nullptr) { 340b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross return startComputeOnCpu(event); 341b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross } else { 342b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross return startComputeOnDevice(event); 343b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross } 344b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross} 345b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 346b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Grossint StepExecutor::startComputeOnDevice(sp<Event>* event) { 347b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross nnAssert(mDriver != nullptr); 348b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross 349425b2594c76e934dfdbc93209253e3c189571149David Gross *event = nullptr; 350425b2594c76e934dfdbc93209253e3c189571149David Gross 351b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross // TODO: Remove the mPreparedModel == nullptr case once we've fully integrated 3521f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // ExecutionPlan with the compilation and execution phases of the NN API 353b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross if (mPreparedModel == nullptr) { 3541f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross Model model; 3551f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mModel->setHidlModel(&model); 3561f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross 3571f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // TODO Dangerous! In async, the model will outlive it here. Safe for now 3581f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross sp<Event> preparationEvent = new Event(); 3591f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross ErrorStatus prepareStatus = ErrorStatus::GENERAL_FAILURE; 3605f916fc9a7ae95f172492bdfe5344c37beff3a6fMichael Butler 361b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mDriver->prepareModel(model, preparationEvent, 362b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross [&prepareStatus, this](ErrorStatus status, 363b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross const sp<IPreparedModel>& prepared) { 364b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross prepareStatus = status; 365b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross mPreparedModel = prepared; 366b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross }); 3675f916fc9a7ae95f172492bdfe5344c37beff3a6fMichael Butler 3681f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // Immediately synchronize with event for now 3691f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross // TODO: change to asynchronous later 3701f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross Event::Status eventStatus = preparationEvent->wait(); 3715f916fc9a7ae95f172492bdfe5344c37beff3a6fMichael Butler 372b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross if (prepareStatus != ErrorStatus::NONE || mPreparedModel == nullptr || 3735f916fc9a7ae95f172492bdfe5344c37beff3a6fMichael Butler eventStatus != Event::Status::SUCCESS) { 3741f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross return ANEURALNETWORKS_OP_FAILED; 3751f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross } 376707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 377707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 378e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // We separate the input & output pools so that we reduce the copying done if we 379e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // do an eventual remoting (hidl_memory->update()). We could also use it to set 380e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // protection on read only memory but that's not currently done. 381e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross Memory inputPointerArguments; 382e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross Memory outputPointerArguments; 383e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross 3848b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // Layout the input and output data 385e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross int n = allocatePointerArgumentsToPool(&mInputs, &inputPointerArguments); 386707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 387707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return n; 388707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 389e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross n = allocatePointerArgumentsToPool(&mOutputs, &outputPointerArguments); 390707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 391707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return n; 392707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 393707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 3948b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // Copy the input data that was specified via a pointer. 395e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // inputPointerArguments.update(); 3968b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& info : mInputs) { 3978b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (info.state == ModelArgumentInfo::POINTER) { 39862cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet DataLocation& loc = info.locationAndLength; 3992150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet uint8_t* data = nullptr; 400e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross int n = inputPointerArguments.getPointer(&data); 4012150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 4022150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet return n; 4032150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet } 4048b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet memcpy(data + loc.offset, info.buffer, loc.length); 4058b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 406707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 407e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross // TODO: Add inputPointerArguments.commit() and .update() at all the right places 408707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 409707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet Request request; 41062cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mInputs, &request.inputs); 41162cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mOutputs, &request.outputs); 4128b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = mMemories.size(); 4138b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet request.pools.resize(count); 4148b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (uint32_t i = 0; i < count; i++) { 4158b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet request.pools[i] = mMemories[i]->getHidlMemory(); 4168b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 417707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 418425b2594c76e934dfdbc93209253e3c189571149David Gross // Prepare the event for asynchronous execution. The sp<Event> object is 419425b2594c76e934dfdbc93209253e3c189571149David Gross // returned when the execution has been successfully launched, otherwise a 420425b2594c76e934dfdbc93209253e3c189571149David Gross // nullptr is returned. The sp is used for ref-counting purposes. Without 421425b2594c76e934dfdbc93209253e3c189571149David Gross // it, the HIDL service could attempt to communicate with a dead event 422425b2594c76e934dfdbc93209253e3c189571149David Gross // object. 423689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // 424689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: Explain the "dead event" problem further, either here or 425689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // in the design document. 426689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler sp<Event> eventSp = new Event(); 427689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 428b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross LOG(DEBUG) << "Before mPreparedModel->execute() " << toString(request); 4293ced3cfd5b8f22b632c35f24e585c4847383b195David Gross // Execute. 430689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: What happens to the Event if the service dies abnormally 431689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // -- won't that keep the Event live forever, because the service 432689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // never has the opportunity to bump the reference count down? Or 433689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // maybe the HIDL infrastructure handles this magically? At worst, 434689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // it seems like this is a small memory leak, if the Event stays 435689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // alive forever. 436b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross if (mPreparedModel->execute(request, eventSp) != ErrorStatus::NONE) { 437707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet LOG(DEBUG) << "**Execute failed**"; 438707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_OP_FAILED; 439707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 440707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 441689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: Remove this synchronization point when the block of code below is 442689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // removed. 443689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler Event::Status status = eventSp->wait(); 444689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler if (status != Event::Status::SUCCESS) { 445689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler LOG(DEBUG) << "**Execute async failed**"; 446689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler return ANEURALNETWORKS_OP_FAILED; 447689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler } 448689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 449707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // Copy the output data from shared memory to the output buffers. 450689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: Move this block of code somewhere else. It should not be in the 451689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // startCompute function. 452689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: outputMemory->update(); outputMemory->commit() 4538b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (auto& info : mOutputs) { 4548b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (info.state == ModelArgumentInfo::POINTER) { 45562cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet DataLocation& loc = info.locationAndLength; 4562150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet uint8_t* data = nullptr; 457e413eef7e5f790acdb8e5d07cacfee9afab6f7beDavid Gross int n = outputPointerArguments.getPointer(&data); 4582150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet if (n != ANEURALNETWORKS_NO_ERROR) { 4592150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet return n; 4602150f1d186b2854fb5aa609594be12a667f845f0Jean-Luc Brouillet } 4618b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet memcpy(info.buffer, data + loc.offset, loc.length); 4628b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 463707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet } 464b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Gross LOG(DEBUG) << "StepExecutor::startComputeOnDevice completed"; 465707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 466425b2594c76e934dfdbc93209253e3c189571149David Gross *event = eventSp; 467707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet return ANEURALNETWORKS_NO_ERROR; 468707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 469707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 470689d892203c06c66c7bb2e374462a8434e40b75fMichael Butlerstatic void asyncStartComputeOnCpu(const Model& model, const Request& request, 471689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler const std::vector<RunTimePoolInfo>& runTimePoolInfos, 472689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler const sp<IEvent>& event) { 473689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler CpuExecutor executor; 474689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler int err = executor.run(model, request, runTimePoolInfos); 4755f916fc9a7ae95f172492bdfe5344c37beff3a6fMichael Butler ErrorStatus status = err == ANEURALNETWORKS_NO_ERROR ? 4765f916fc9a7ae95f172492bdfe5344c37beff3a6fMichael Butler ErrorStatus::NONE : ErrorStatus::GENERAL_FAILURE; 4775f916fc9a7ae95f172492bdfe5344c37beff3a6fMichael Butler event->notify(status); 478689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler} 479689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 480b26049114bc4c64e6bea3a5d5d129fcaec8e69b6David Grossint StepExecutor::startComputeOnCpu(sp<Event>* event) { 481707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet // TODO: use a thread pool 482689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 4831f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross Model model; 4841f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross mModel->setHidlModel(&model); 4851f4381539b7e89c42336ee7cd1addb9a4c317b34David Gross 486689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // Prepare the event for asynchronous execution. The sp<Event> object is 487425b2594c76e934dfdbc93209253e3c189571149David Gross // returned when the execution has been successfully launched, otherwise a 488425b2594c76e934dfdbc93209253e3c189571149David Gross // nullptr is returned. 489689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler sp<Event> eventSp = new Event(); 490425b2594c76e934dfdbc93209253e3c189571149David Gross *event = nullptr; 491707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 492707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet std::vector<RunTimePoolInfo> runTimePoolInfos; 4938b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet uint32_t count = mMemories.size(); 4948b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet runTimePoolInfos.resize(count); 4958b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (uint32_t i = 0; i < count; i++) { 4968b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet const Memory* mem = mMemories[i]; 4978b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet runTimePoolInfos[i].set(mem->getHidlMemory()); 4988b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 4998b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet // Create as many pools as there are input / output. 5008b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet auto fixPointerArguments = [&runTimePoolInfos](std::vector<ModelArgumentInfo>& argumentInfos) { 5018b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet for (ModelArgumentInfo& argumentInfo : argumentInfos) { 5028b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet if (argumentInfo.state == ModelArgumentInfo::POINTER) { 503389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet RunTimePoolInfo runTimeInfo = { 504389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet .buffer = static_cast<uint8_t*>(argumentInfo.buffer)}; 50562cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet argumentInfo.locationAndLength.poolIndex = 506389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet static_cast<uint32_t>(runTimePoolInfos.size()); 50762cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet argumentInfo.locationAndLength.offset = 0; 5088b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet runTimePoolInfos.push_back(runTimeInfo); 5098b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 5108b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet } 5118b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet }; 5128b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet fixPointerArguments(mInputs); 5138b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet fixPointerArguments(mOutputs); 514707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 5158b99bb1d98a42b67ba1c00e12c7abb3708cf7c05Jean-Luc Brouillet Request request; 51662cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mInputs, &request.inputs); 51762cc2758c1c2d303861e209f26bddcf4d7564b73Jean-Luc Brouillet setRequestArgumentArray(mOutputs, &request.outputs); 518707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 519689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler // TODO: should model be moved with a std::cref? 520689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler std::thread thread(asyncStartComputeOnCpu, model, std::move(request), 521689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler std::move(runTimePoolInfos), eventSp); 522689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler eventSp->bind_thread(std::move(thread)); 523689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler 524425b2594c76e934dfdbc93209253e3c189571149David Gross *event = eventSp; 525689d892203c06c66c7bb2e374462a8434e40b75fMichael Butler return ANEURALNETWORKS_NO_ERROR; 526707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet} 527707dbd2d55f5dacf78ffb3ad7c8b3f37c2e9d758Jean-Luc Brouillet 528389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} // namespace nn 529389f26c7c442c37994db9f43d013fe3953c9353cJean-Luc Brouillet} // namespace android 530