12c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung/* 22c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung * Copyright (C) 2017 The Android Open Source Project 32c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung * 42c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung * Licensed under the Apache License, Version 2.0 (the "License"); 52c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung * you may not use this file except in compliance with the License. 62c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung * You may obtain a copy of the License at 72c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung * 82c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung * http://www.apache.org/licenses/LICENSE-2.0 92c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung * 102c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung * Unless required by applicable law or agreed to in writing, software 112c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung * distributed under the License is distributed on an "AS IS" BASIS, 122c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 132c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung * See the License for the specific language governing permissions and 142c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung * limitations under the License. 152c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung */ 162c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung 17cf22a57c1a05272d055b0deaa10094852dece797Michael Butler#include "Callbacks.h" 182c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung#include "TestHarness.h" 19a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang#include "Utils.h" 202c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung 212c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung#include <android-base/logging.h> 22a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang#include <android/hardware/neuralnetworks/1.0/IDevice.h> 23a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang#include <android/hardware/neuralnetworks/1.0/IExecutionCallback.h> 24a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang#include <android/hardware/neuralnetworks/1.0/IPreparedModel.h> 25a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang#include <android/hardware/neuralnetworks/1.0/IPreparedModelCallback.h> 26a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang#include <android/hardware/neuralnetworks/1.0/types.h> 27a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang#include <android/hidl/allocator/1.0/IAllocator.h> 282c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung#include <android/hidl/memory/1.0/IMemory.h> 292c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung#include <hidlmemory/mapping.h> 300897ab3aad7d062c101892fd40a96271714f1613Michael Butler#include <iostream> 312c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung 322c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sungnamespace android { 332c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sungnamespace hardware { 342c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sungnamespace neuralnetworks { 352c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung 362c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sungnamespace generated_tests { 37cf22a57c1a05272d055b0deaa10094852dece797Michael Butlerusing ::android::hardware::neuralnetworks::V1_0::implementation::ExecutionCallback; 38cf22a57c1a05272d055b0deaa10094852dece797Michael Butlerusing ::android::hardware::neuralnetworks::V1_0::implementation::PreparedModelCallback; 39de16694a6079355867287e4613ae89123c224171Mika Raentousing ::test_helper::filter; 40de16694a6079355867287e4613ae89123c224171Mika Raentousing ::test_helper::for_all; 41de16694a6079355867287e4613ae89123c224171Mika Raentousing ::test_helper::for_each; 42de16694a6079355867287e4613ae89123c224171Mika Raentousing ::test_helper::resize_accordingly; 43de16694a6079355867287e4613ae89123c224171Mika Raentousing ::test_helper::MixedTyped; 44de16694a6079355867287e4613ae89123c224171Mika Raentousing ::test_helper::MixedTypedExampleType; 45de16694a6079355867287e4613ae89123c224171Mika Raentousing ::test_helper::Float32Operands; 46de16694a6079355867287e4613ae89123c224171Mika Raentousing ::test_helper::Int32Operands; 47de16694a6079355867287e4613ae89123c224171Mika Raentousing ::test_helper::Quant8Operands; 48de16694a6079355867287e4613ae89123c224171Mika Raentousing ::test_helper::compare; 49f6b85506fecb7ff2b66e47f451443d8f557c8cb0I-Jui (Ray) Sung 505bf4edfb9fb7b1e0893609058dcfc8ff67be17cbI-Jui (Ray) Sungtemplate <typename T> 51f6b85506fecb7ff2b66e47f451443d8f557c8cb0I-Jui (Ray) Sungvoid copy_back_(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) { 52f6b85506fecb7ff2b66e47f451443d8f557c8cb0I-Jui (Ray) Sung MixedTyped& test = *dst; 535bf4edfb9fb7b1e0893609058dcfc8ff67be17cbI-Jui (Ray) Sung for_each<T>(test, [&ra, src](int index, std::vector<T>& m) { 545bf4edfb9fb7b1e0893609058dcfc8ff67be17cbI-Jui (Ray) Sung ASSERT_EQ(m.size(), ra[index].location.length / sizeof(T)); 55f6b85506fecb7ff2b66e47f451443d8f557c8cb0I-Jui (Ray) Sung char* begin = src + ra[index].location.offset; 56f6b85506fecb7ff2b66e47f451443d8f557c8cb0I-Jui (Ray) Sung memcpy(m.data(), begin, ra[index].location.length); 57f6b85506fecb7ff2b66e47f451443d8f557c8cb0I-Jui (Ray) Sung }); 58f6b85506fecb7ff2b66e47f451443d8f557c8cb0I-Jui (Ray) Sung} 59f6b85506fecb7ff2b66e47f451443d8f557c8cb0I-Jui (Ray) Sung 60f6b85506fecb7ff2b66e47f451443d8f557c8cb0I-Jui (Ray) Sungvoid copy_back(MixedTyped* dst, const std::vector<RequestArgument>& ra, char* src) { 61f6b85506fecb7ff2b66e47f451443d8f557c8cb0I-Jui (Ray) Sung copy_back_<float>(dst, ra, src); 62f6b85506fecb7ff2b66e47f451443d8f557c8cb0I-Jui (Ray) Sung copy_back_<int32_t>(dst, ra, src); 63f6b85506fecb7ff2b66e47f451443d8f557c8cb0I-Jui (Ray) Sung copy_back_<uint8_t>(dst, ra, src); 64f6b85506fecb7ff2b66e47f451443d8f557c8cb0I-Jui (Ray) Sung} 65f6b85506fecb7ff2b66e47f451443d8f557c8cb0I-Jui (Ray) Sung 662c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung// Top level driver for models and examples generated by test_generator.py 672c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung// Test driver for those generated from ml/nn/runtime/test/spec 68a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wangvoid EvaluatePreparedModel(sp<IPreparedModel>& preparedModel, std::function<bool(int)> is_ignored, 69842e1a03319361a984cf62e51cbf0486d5f15c46Miao Wang const std::vector<MixedTypedExampleType>& examples, 70842e1a03319361a984cf62e51cbf0486d5f15c46Miao Wang float fpRange = 1e-5f) { 712c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung const uint32_t INPUT = 0; 722c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung const uint32_t OUTPUT = 1; 732c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung 742c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung int example_no = 1; 752c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung for (auto& example : examples) { 762c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung SCOPED_TRACE(example_no++); 772c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung 782c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung const MixedTyped& inputs = example.first; 792c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung const MixedTyped& golden = example.second; 802c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung 812c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung std::vector<RequestArgument> inputs_info, outputs_info; 822c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung uint32_t inputSize = 0, outputSize = 0; 832c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung 842c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung // This function only partially specifies the metadata (vector of RequestArguments). 852c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung // The contents are copied over below. 862c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung for_all(inputs, [&inputs_info, &inputSize](int index, auto, auto s) { 872c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung if (inputs_info.size() <= static_cast<size_t>(index)) inputs_info.resize(index + 1); 882c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung RequestArgument arg = { 892c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung .location = {.poolIndex = INPUT, .offset = 0, .length = static_cast<uint32_t>(s)}, 902c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung .dimensions = {}, 912c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung }; 92959cd78e6766626d4cd9c52f7cdc156fd7880169I-Jui (Ray) Sung RequestArgument arg_empty = { 93959cd78e6766626d4cd9c52f7cdc156fd7880169I-Jui (Ray) Sung .hasNoValue = true, 94959cd78e6766626d4cd9c52f7cdc156fd7880169I-Jui (Ray) Sung }; 95959cd78e6766626d4cd9c52f7cdc156fd7880169I-Jui (Ray) Sung inputs_info[index] = s ? arg : arg_empty; 962c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung inputSize += s; 972c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung }); 982c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung // Compute offset for inputs 1 and so on 992c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung { 1002c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung size_t offset = 0; 1012c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung for (auto& i : inputs_info) { 102959cd78e6766626d4cd9c52f7cdc156fd7880169I-Jui (Ray) Sung if (!i.hasNoValue) i.location.offset = offset; 1032c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung offset += i.location.length; 1042c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung } 1052c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung } 1062c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung 1072c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung MixedTyped test; // holding test results 1082c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung 1092c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung // Go through all outputs, initialize RequestArgument descriptors 110f6b85506fecb7ff2b66e47f451443d8f557c8cb0I-Jui (Ray) Sung resize_accordingly(golden, test); 1112c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung for_all(golden, [&outputs_info, &outputSize](int index, auto, auto s) { 1122c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung if (outputs_info.size() <= static_cast<size_t>(index)) outputs_info.resize(index + 1); 1132c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung RequestArgument arg = { 1142c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung .location = {.poolIndex = OUTPUT, .offset = 0, .length = static_cast<uint32_t>(s)}, 1152c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung .dimensions = {}, 1162c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung }; 1172c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung outputs_info[index] = arg; 1182c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung outputSize += s; 1192c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung }); 1202c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung // Compute offset for outputs 1 and so on 1212c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung { 1222c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung size_t offset = 0; 1232c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung for (auto& i : outputs_info) { 1242c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung i.location.offset = offset; 1252c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung offset += i.location.length; 1262c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung } 1272c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung } 128a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang std::vector<hidl_memory> pools = {nn::allocateSharedMemory(inputSize), 129a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang nn::allocateSharedMemory(outputSize)}; 1302c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung ASSERT_NE(0ull, pools[INPUT].size()); 1312c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung ASSERT_NE(0ull, pools[OUTPUT].size()); 1322c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung 1332c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung // load data 1342c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung sp<IMemory> inputMemory = mapMemory(pools[INPUT]); 1352c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung sp<IMemory> outputMemory = mapMemory(pools[OUTPUT]); 1362c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung ASSERT_NE(nullptr, inputMemory.get()); 1372c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung ASSERT_NE(nullptr, outputMemory.get()); 1382c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung char* inputPtr = reinterpret_cast<char*>(static_cast<void*>(inputMemory->getPointer())); 1392c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung char* outputPtr = reinterpret_cast<char*>(static_cast<void*>(outputMemory->getPointer())); 1402c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung ASSERT_NE(nullptr, inputPtr); 1412c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung ASSERT_NE(nullptr, outputPtr); 1422c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung inputMemory->update(); 1432c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung outputMemory->update(); 1442c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung 1452c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung // Go through all inputs, copy the values 1462c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung for_all(inputs, [&inputs_info, inputPtr](int index, auto p, auto s) { 1472c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung char* begin = (char*)p; 1482c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung char* end = begin + s; 1492c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung // TODO: handle more than one input 1502c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung std::copy(begin, end, inputPtr + inputs_info[index].location.offset); 1512c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung }); 1522c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung 1532c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung inputMemory->commit(); 1542c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung outputMemory->commit(); 155cf22a57c1a05272d055b0deaa10094852dece797Michael Butler 156cf22a57c1a05272d055b0deaa10094852dece797Michael Butler // launch execution 157cf22a57c1a05272d055b0deaa10094852dece797Michael Butler sp<ExecutionCallback> executionCallback = new ExecutionCallback(); 158cf22a57c1a05272d055b0deaa10094852dece797Michael Butler ASSERT_NE(nullptr, executionCallback.get()); 159cf22a57c1a05272d055b0deaa10094852dece797Michael Butler Return<ErrorStatus> executionLaunchStatus = preparedModel->execute( 160cf22a57c1a05272d055b0deaa10094852dece797Michael Butler {.inputs = inputs_info, .outputs = outputs_info, .pools = pools}, executionCallback); 161cf22a57c1a05272d055b0deaa10094852dece797Michael Butler ASSERT_TRUE(executionLaunchStatus.isOk()); 162cf22a57c1a05272d055b0deaa10094852dece797Michael Butler EXPECT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(executionLaunchStatus)); 163cf22a57c1a05272d055b0deaa10094852dece797Michael Butler 164cf22a57c1a05272d055b0deaa10094852dece797Michael Butler // retrieve execution status 165cf22a57c1a05272d055b0deaa10094852dece797Michael Butler executionCallback->wait(); 166cf22a57c1a05272d055b0deaa10094852dece797Michael Butler ErrorStatus executionReturnStatus = executionCallback->getStatus(); 167cf22a57c1a05272d055b0deaa10094852dece797Michael Butler EXPECT_EQ(ErrorStatus::NONE, executionReturnStatus); 1682c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung 1692c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung // validate results 1702c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung outputMemory->read(); 171f6b85506fecb7ff2b66e47f451443d8f557c8cb0I-Jui (Ray) Sung copy_back(&test, outputs_info, outputPtr); 1722c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung outputMemory->commit(); 1737d765bdd0f747a9897310040f81f670f6ef3e3f3I-Jui (Ray) Sung // Filter out don't cares 1745bf4edfb9fb7b1e0893609058dcfc8ff67be17cbI-Jui (Ray) Sung MixedTyped filtered_golden = filter(golden, is_ignored); 1755bf4edfb9fb7b1e0893609058dcfc8ff67be17cbI-Jui (Ray) Sung MixedTyped filtered_test = filter(test, is_ignored); 1767d765bdd0f747a9897310040f81f670f6ef3e3f3I-Jui (Ray) Sung 1772c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung // We want "close-enough" results for float 178842e1a03319361a984cf62e51cbf0486d5f15c46Miao Wang compare(filtered_golden, filtered_test, fpRange); 1792c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung } 1802c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung} 1812c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung 182f76acd0312f7d47bd2e371f027a54bca581d8f8fMichael Butlervoid Execute(const sp<V1_0::IDevice>& device, std::function<V1_0::Model(void)> create_model, 183a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang std::function<bool(int)> is_ignored, 184a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang const std::vector<MixedTypedExampleType>& examples) { 185a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang V1_0::Model model = create_model(); 186a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang 187a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang // see if service can handle model 188a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang bool fullySupportsModel = false; 189a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang Return<void> supportedCall = device->getSupportedOperations( 1904d5bb1097a34495212c09473b477dc97acb99264Michael Butler model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) { 1914d5bb1097a34495212c09473b477dc97acb99264Michael Butler ASSERT_EQ(ErrorStatus::NONE, status); 192a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang ASSERT_NE(0ul, supported.size()); 193a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang fullySupportsModel = 194a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; }); 195a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang }); 196a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang ASSERT_TRUE(supportedCall.isOk()); 1974d5bb1097a34495212c09473b477dc97acb99264Michael Butler 1984d5bb1097a34495212c09473b477dc97acb99264Michael Butler // launch prepare model 1994d5bb1097a34495212c09473b477dc97acb99264Michael Butler sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback(); 2004d5bb1097a34495212c09473b477dc97acb99264Michael Butler ASSERT_NE(nullptr, preparedModelCallback.get()); 201a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang Return<ErrorStatus> prepareLaunchStatus = device->prepareModel(model, preparedModelCallback); 202a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang ASSERT_TRUE(prepareLaunchStatus.isOk()); 2034d5bb1097a34495212c09473b477dc97acb99264Michael Butler ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus)); 204a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang 205a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang // retrieve prepared model 206a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang preparedModelCallback->wait(); 207a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); 208a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel(); 209a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang 210a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang // early termination if vendor service cannot fully prepare model 2114d5bb1097a34495212c09473b477dc97acb99264Michael Butler if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) { 212a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang ASSERT_EQ(nullptr, preparedModel.get()); 213a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " 214a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang "prepare model that it does not support."; 215a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang std::cout << "[ ] Early termination of test because vendor service cannot " 216a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang "prepare model that it does not support." 217a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang << std::endl; 218a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang return; 219a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang } 2204d5bb1097a34495212c09473b477dc97acb99264Michael Butler EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); 221a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang ASSERT_NE(nullptr, preparedModel.get()); 222a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang 223a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang EvaluatePreparedModel(preparedModel, is_ignored, examples); 224a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang} 225a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang 226f76acd0312f7d47bd2e371f027a54bca581d8f8fMichael Butlervoid Execute(const sp<V1_1::IDevice>& device, std::function<V1_1::Model(void)> create_model, 227a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang std::function<bool(int)> is_ignored, 228a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang const std::vector<MixedTypedExampleType>& examples) { 229a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang V1_1::Model model = create_model(); 230a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang 231a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang // see if service can handle model 232a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang bool fullySupportsModel = false; 233a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang Return<void> supportedCall = device->getSupportedOperations_1_1( 2344d5bb1097a34495212c09473b477dc97acb99264Michael Butler model, [&fullySupportsModel](ErrorStatus status, const hidl_vec<bool>& supported) { 2354d5bb1097a34495212c09473b477dc97acb99264Michael Butler ASSERT_EQ(ErrorStatus::NONE, status); 236a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang ASSERT_NE(0ul, supported.size()); 237a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang fullySupportsModel = 238a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang std::all_of(supported.begin(), supported.end(), [](bool valid) { return valid; }); 239a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang }); 240a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang ASSERT_TRUE(supportedCall.isOk()); 2414d5bb1097a34495212c09473b477dc97acb99264Michael Butler 2424d5bb1097a34495212c09473b477dc97acb99264Michael Butler // launch prepare model 2434d5bb1097a34495212c09473b477dc97acb99264Michael Butler sp<PreparedModelCallback> preparedModelCallback = new PreparedModelCallback(); 2444d5bb1097a34495212c09473b477dc97acb99264Michael Butler ASSERT_NE(nullptr, preparedModelCallback.get()); 2452504c2fe4f3dc6975165d35a8ea1610d8bc4c477Michael Butler Return<ErrorStatus> prepareLaunchStatus = device->prepareModel_1_1( 2462504c2fe4f3dc6975165d35a8ea1610d8bc4c477Michael Butler model, ExecutionPreference::FAST_SINGLE_ANSWER, preparedModelCallback); 247a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang ASSERT_TRUE(prepareLaunchStatus.isOk()); 2484d5bb1097a34495212c09473b477dc97acb99264Michael Butler ASSERT_EQ(ErrorStatus::NONE, static_cast<ErrorStatus>(prepareLaunchStatus)); 249a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang 250a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang // retrieve prepared model 251a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang preparedModelCallback->wait(); 252a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang ErrorStatus prepareReturnStatus = preparedModelCallback->getStatus(); 253a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang sp<IPreparedModel> preparedModel = preparedModelCallback->getPreparedModel(); 254a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang 255a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang // early termination if vendor service cannot fully prepare model 2564d5bb1097a34495212c09473b477dc97acb99264Michael Butler if (!fullySupportsModel && prepareReturnStatus != ErrorStatus::NONE) { 257a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang ASSERT_EQ(nullptr, preparedModel.get()); 258a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang LOG(INFO) << "NN VTS: Early termination of test because vendor service cannot " 259a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang "prepare model that it does not support."; 260a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang std::cout << "[ ] Early termination of test because vendor service cannot " 261a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang "prepare model that it does not support." 262a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang << std::endl; 263a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang return; 264a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang } 2654d5bb1097a34495212c09473b477dc97acb99264Michael Butler EXPECT_EQ(ErrorStatus::NONE, prepareReturnStatus); 266a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang ASSERT_NE(nullptr, preparedModel.get()); 267a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang 268842e1a03319361a984cf62e51cbf0486d5f15c46Miao Wang // If in relaxed mode, set the error range to be 5ULP of FP16. 269842e1a03319361a984cf62e51cbf0486d5f15c46Miao Wang float fpRange = !model.relaxComputationFloat32toFloat16 ? 1e-5f : 5.0f * 0.0009765625f; 270842e1a03319361a984cf62e51cbf0486d5f15c46Miao Wang EvaluatePreparedModel(preparedModel, is_ignored, examples, fpRange); 271a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang} 272a2d04c828e98bdadc6dd44c6235556451e4e2a88Miao Wang 2732c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung} // namespace generated_tests 2742c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung 2752c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung} // namespace neuralnetworks 2762c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung} // namespace hardware 2772c4e1368e1dacbe9fa3d8e696e713bb08f09953eI-Jui (Ray) Sung} // namespace android 278