1242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung/* 2242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung * Copyright (C) 2017 The Android Open Source Project 3242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung * 4242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung * Licensed under the Apache License, Version 2.0 (the "License"); 5242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung * you may not use this file except in compliance with the License. 6242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung * You may obtain a copy of the License at 7242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung * 8242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung * http://www.apache.org/licenses/LICENSE-2.0 9242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung * 10242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung * Unless required by applicable law or agreed to in writing, software 11242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung * distributed under the License is distributed on an "AS IS" BASIS, 12242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung * See the License for the specific language governing permissions and 14242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung * limitations under the License. 15242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung */ 16242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung 17ad36b7a0dd7468bbe6a0433a52cbf89561ff5538I-Jui (Ray) Sung// Top level driver for models and examples generated by test_generator.py 18242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung 19663155d58ca1e1eb42e01495e236258e0c00d40fDavid Gross#include "Bridge.h" 20242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung#include "NeuralNetworksWrapper.h" 216a0d306cf902e13ab147c7533b2cb02540ee66d5I-Jui (Ray) Sung#include "TestHarness.h" 22242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung 23242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung#include <gtest/gtest.h> 24242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung#include <cassert> 25242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung#include <cmath> 264062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung#include <fstream> 27242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung#include <iostream> 28242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung#include <map> 29242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung 30663155d58ca1e1eb42e01495e236258e0c00d40fDavid Gross// Uncomment the following line to generate DOT graphs. 31663155d58ca1e1eb42e01495e236258e0c00d40fDavid Gross// 32663155d58ca1e1eb42e01495e236258e0c00d40fDavid Gross// #define GRAPH GRAPH 33663155d58ca1e1eb42e01495e236258e0c00d40fDavid Gross 346a0d306cf902e13ab147c7533b2cb02540ee66d5I-Jui (Ray) Sungnamespace generated_tests { 35ad36b7a0dd7468bbe6a0433a52cbf89561ff5538I-Jui (Ray) Sungusing namespace android::nn::wrapper; 36951a1eeb114a6aacc87d6df67b9090bb94afe3bfMika Raentousing namespace test_helper; 376a0d306cf902e13ab147c7533b2cb02540ee66d5I-Jui (Ray) Sung 38663155d58ca1e1eb42e01495e236258e0c00d40fDavid Grossvoid graphDump([[maybe_unused]] const char* name, [[maybe_unused]] const Model& model) { 39663155d58ca1e1eb42e01495e236258e0c00d40fDavid Gross#ifdef GRAPH 40663155d58ca1e1eb42e01495e236258e0c00d40fDavid Gross ::android::nn::bridge_tests::graphDump( 41663155d58ca1e1eb42e01495e236258e0c00d40fDavid Gross name, 42663155d58ca1e1eb42e01495e236258e0c00d40fDavid Gross reinterpret_cast<const ::android::nn::ModelBuilder*>(model.getHandle())); 43663155d58ca1e1eb42e01495e236258e0c00d40fDavid Gross#endif 44663155d58ca1e1eb42e01495e236258e0c00d40fDavid Gross} 45663155d58ca1e1eb42e01495e236258e0c00d40fDavid Gross 46ad36b7a0dd7468bbe6a0433a52cbf89561ff5538I-Jui (Ray) Sungtemplate <typename T> 474062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sungstatic void print(std::ostream& os, const MixedTyped& test) { 484062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung // dump T-typed inputs 494062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung for_each<T>(test, [&os](int idx, const std::vector<T>& f) { 504062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung os << " aliased_output" << idx << ": ["; 514062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung for (size_t i = 0; i < f.size(); ++i) { 524062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung os << (i == 0 ? "" : ", ") << +f[i]; 536a0d306cf902e13ab147c7533b2cb02540ee66d5I-Jui (Ray) Sung } 544062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung os << "],\n"; 554062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung }); 564062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung} 574062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung 584062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sungstatic void printAll(std::ostream& os, const MixedTyped& test) { 594062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung print<float>(os, test); 604062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung print<int32_t>(os, test); 614062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung print<uint8_t>(os, test); 624062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung} 634062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung 644062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung// Test driver for those generated from ml/nn/runtime/test/spec 654062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sungstatic void execute(std::function<void(Model*)> createModel, 664062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung std::function<bool(int)> isIgnored, 674062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung std::vector<MixedTypedExampleType>& examples, 684062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung std::string dumpFile = "") { 694062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung Model model; 704062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung createModel(&model); 714062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung model.finish(); 72663155d58ca1e1eb42e01495e236258e0c00d40fDavid Gross graphDump("", model); 734062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung bool dumpToFile = !dumpFile.empty(); 744062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung 754062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung std::ofstream s; 764062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung if (dumpToFile) { 774062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung s.open(dumpFile, std::ofstream::trunc); 784062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung ASSERT_TRUE(s.is_open()); 796a0d306cf902e13ab147c7533b2cb02540ee66d5I-Jui (Ray) Sung } 804062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung 814062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung int exampleNo = 0; 824062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung Compilation compilation(&model); 834062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung compilation.finish(); 84b74d2837ab1687c1a4f913aa5f90a9838efe0addMiao Wang 85b74d2837ab1687c1a4f913aa5f90a9838efe0addMiao Wang // If in relaxed mode, set the error range to be 5ULP of FP16. 86b74d2837ab1687c1a4f913aa5f90a9838efe0addMiao Wang float fpRange = !model.isRelaxed() ? 1e-5f : 5.0f * 0.0009765625f; 874062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung for (auto& example : examples) { 884062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung SCOPED_TRACE(exampleNo); 894062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung // TODO: We leave it as a copy here. 904062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung // Should verify if the input gets modified by the test later. 914062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung MixedTyped inputs = example.first; 924062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung const MixedTyped& golden = example.second; 934062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung 944062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung Execution execution(&compilation); 954062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung 964062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung // Set all inputs 974062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung for_all(inputs, [&execution](int idx, const void* p, size_t s) { 9843d018d1a66872031568857fa369024c9640a085Miao Wang const void* buffer = s == 0 ? nullptr : p; 9943d018d1a66872031568857fa369024c9640a085Miao Wang ASSERT_EQ(Result::NO_ERROR, execution.setInput(idx, buffer, s)); 1004062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung }); 1014062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung 1024062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung MixedTyped test; 1034062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung // Go through all typed outputs 1044062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung resize_accordingly(golden, test); 1054062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung for_all(test, [&execution](int idx, void* p, size_t s) { 10643d018d1a66872031568857fa369024c9640a085Miao Wang void* buffer = s == 0 ? nullptr : p; 10743d018d1a66872031568857fa369024c9640a085Miao Wang ASSERT_EQ(Result::NO_ERROR, execution.setOutput(idx, buffer, s)); 1084062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung }); 1094062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung 1104062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung Result r = execution.compute(); 1114062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung ASSERT_EQ(Result::NO_ERROR, r); 1124062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung 1134062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung // Dump all outputs for the slicing tool 1144062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung if (dumpToFile) { 1154062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung s << "output" << exampleNo << " = {\n"; 1164062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung printAll(s, test); 1174062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung // all outputs are done 1184062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung s << "}\n"; 1194062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung } 1204062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung 1214062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung // Filter out don't cares 1224062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung MixedTyped filteredGolden = filter(golden, isIgnored); 1234062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung MixedTyped filteredTest = filter(test, isIgnored); 1244062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung // We want "close-enough" results for float 125b74d2837ab1687c1a4f913aa5f90a9838efe0addMiao Wang 126b74d2837ab1687c1a4f913aa5f90a9838efe0addMiao Wang compare(filteredGolden, filteredTest, fpRange); 1274062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung exampleNo++; 1284062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung } 1294062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung} 1304062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung 1316a0d306cf902e13ab147c7533b2cb02540ee66d5I-Jui (Ray) Sung}; // namespace generated_tests 132ad36b7a0dd7468bbe6a0433a52cbf89561ff5538I-Jui (Ray) Sung 1336a0d306cf902e13ab147c7533b2cb02540ee66d5I-Jui (Ray) Sungusing namespace android::nn::wrapper; 1344062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sung 1356a0d306cf902e13ab147c7533b2cb02540ee66d5I-Jui (Ray) Sung// Mixed-typed examples 136951a1eeb114a6aacc87d6df67b9090bb94afe3bfMika Raentotypedef test_helper::MixedTypedExampleType MixedTypedExample; 137242c6dc1f314646f1a87c66140f26d7623cc399aI-Jui (Ray) Sung 1386a0d306cf902e13ab147c7533b2cb02540ee66d5I-Jui (Ray) Sungclass GeneratedTests : public ::testing::Test { 139951a1eeb114a6aacc87d6df67b9090bb94afe3bfMika Raentoprotected: 140820215d28bed6c90f696cde0f282445d16da432eMiao Wang virtual void SetUp() {} 141dcd2fbf1da7ebdc1aa1b57c74db4fffd2911e3b8I-Jui (Ray) Sung}; 142dcd2fbf1da7ebdc1aa1b57c74db4fffd2911e3b8I-Jui (Ray) Sung 1436a0d306cf902e13ab147c7533b2cb02540ee66d5I-Jui (Ray) Sung// Testcases generated from runtime/test/specs/*.mod.py 144951a1eeb114a6aacc87d6df67b9090bb94afe3bfMika Raentousing namespace test_helper; 1454062279c74f2c995349b7ff04f22feb89eb81e47I-Jui (Ray) Sungusing namespace generated_tests; 1466a0d306cf902e13ab147c7533b2cb02540ee66d5I-Jui (Ray) Sung#include "generated/all_generated_tests.cpp" 1476a0d306cf902e13ab147c7533b2cb02540ee66d5I-Jui (Ray) Sung// End of testcases generated from runtime/test/specs/*.mod.py 148