unit_test_launcher.cc revision 4e180b6a0b4720a9b8e9e959a882386f690f08ff
13551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)// Copyright 2013 The Chromium Authors. All rights reserved. 23551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)// Use of this source code is governed by a BSD-style license that can be 33551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)// found in the LICENSE file. 43551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 54e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/test/launcher/unit_test_launcher.h" 63551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 7424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)#include "base/bind.h" 8424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)#include "base/callback_helpers.h" 93551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)#include "base/command_line.h" 103551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)#include "base/compiler_specific.h" 113551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)#include "base/file_util.h" 124e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/files/scoped_temp_dir.h" 133551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)#include "base/format_macros.h" 1458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)#include "base/message_loop/message_loop.h" 153551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)#include "base/stl_util.h" 1658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)#include "base/strings/string_number_conversions.h" 173551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)#include "base/strings/string_util.h" 1858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)#include "base/sys_info.h" 193551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)#include "base/test/gtest_xml_util.h" 204e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/test/launcher/parallel_test_launcher.h" 214e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)#include "base/test/launcher/test_launcher.h" 22424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)#include "base/test/test_switches.h" 233551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)#include "base/test/test_timeouts.h" 2458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)#include "base/threading/thread_checker.h" 253551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)#include "testing/gtest/include/gtest/gtest.h" 263551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 273551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)namespace base { 283551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 293551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)namespace { 303551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 3158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)// This constant controls how many tests are run in a single batch by default. 3258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)const size_t kDefaultTestBatchLimit = 10; 333551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 34d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)const char kHelpFlag[] = "help"; 35d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) 363551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)// Flag to enable the new launcher logic. 373551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)// TODO(phajdan.jr): Remove it, http://crbug.com/236893 . 383551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)const char kBraveNewTestLauncherFlag[] = "brave-new-test-launcher"; 393551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 403551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)// Flag to run all tests in a single process. 413551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)const char kSingleProcessTestsFlag[] = "single-process-tests"; 423551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 43d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)void PrintUsage() { 44d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) fprintf(stdout, 45d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) "Runs tests using the gtest framework, each batch of tests being\n" 46d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) "run in their own process. Supported command-line flags:\n" 47d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) "\n" 48d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) " --single-process-tests\n" 49d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) " Runs the tests and the launcher in the same process. Useful\n" 50d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) " for debugging a specific test in a debugger.\n" 51d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) " --test-launcher-jobs=N\n" 52d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) " Sets the number of parallel test jobs to N.\n" 53d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) " --test-launcher-batch-limit=N\n" 54d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) " Sets the limit of test batch to run in a single process to N.\n" 55d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) " --gtest_filter=...\n" 56d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) " Runs a subset of tests (see --gtest_help for more info).\n" 57d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) " --help\n" 58d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) " Shows this message.\n" 59d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) " --gtest_help\n" 60d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) " Shows the gtest help message.\n"); 61d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) fflush(stdout); 62d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)} 63d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) 643551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)// Returns command line for child GTest process based on the command line 653551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)// of current process. |test_names| is a vector of test full names 663551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)// (e.g. "A.B"), |output_file| is path to the GTest XML output file. 673551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)CommandLine GetCommandLineForChildGTestProcess( 683551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) const std::vector<std::string>& test_names, 693551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) const base::FilePath& output_file) { 70424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) CommandLine new_cmd_line(*CommandLine::ForCurrentProcess()); 713551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 72424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) new_cmd_line.AppendSwitchPath(switches::kTestLauncherOutput, output_file); 73424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) new_cmd_line.AppendSwitchASCII(kGTestFilterFlag, JoinString(test_names, ":")); 743551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) new_cmd_line.AppendSwitch(kSingleProcessTestsFlag); 753551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) new_cmd_line.AppendSwitch(kBraveNewTestLauncherFlag); 763551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 773551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return new_cmd_line; 783551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)} 793551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 803551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)class UnitTestLauncherDelegate : public TestLauncherDelegate { 8158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) public: 8258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) UnitTestLauncherDelegate(size_t jobs, size_t batch_limit) 8358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) : parallel_launcher_(jobs), 8458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) batch_limit_(batch_limit) { 8558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) } 8658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 8758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) virtual ~UnitTestLauncherDelegate() { 8858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 8958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) } 9058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 913551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) private: 923551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) struct TestLaunchInfo { 933551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) std::string GetFullName() const { 943551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return test_case_name + "." + test_name; 953551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) } 963551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 973551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) std::string test_case_name; 983551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) std::string test_name; 993551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) TestResultCallback callback; 1003551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) }; 1013551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 1024e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) virtual void OnTestIterationStarting() OVERRIDE { 1034e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) // Nothing to do. 1044e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 1054e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 10668043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) virtual std::string GetTestNameForFiltering( 10768043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) const testing::TestCase* test_case, 10868043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) const testing::TestInfo* test_info) OVERRIDE { 10968043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 11068043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) 11168043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) return std::string(test_case->name()) + "." + test_info->name(); 11268043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) } 11368043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) 1143551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) virtual bool ShouldRunTest(const testing::TestCase* test_case, 1153551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) const testing::TestInfo* test_info) OVERRIDE { 11658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 11758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 1183551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // There is no additional logic to disable specific tests. 1193551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return true; 1203551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) } 1213551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 1223551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) virtual void RunTest(const testing::TestCase* test_case, 1233551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) const testing::TestInfo* test_info, 1243551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) const TestResultCallback& callback) OVERRIDE { 12558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 12658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 1273551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) TestLaunchInfo launch_info; 1283551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) launch_info.test_case_name = test_case->name(); 1293551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) launch_info.test_name = test_info->name(); 1303551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) launch_info.callback = callback; 1313551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) tests_.push_back(launch_info); 1323551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 1333551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // Run tests in batches no larger than the limit. 13458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) if (tests_.size() >= batch_limit_) 13558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) RunRemainingTests(); 1363551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) } 1373551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 13858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) virtual void RunRemainingTests() OVERRIDE { 13958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 140424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) 1413551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) if (tests_.empty()) 1423551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return; 1433551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 1443551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // Create a dedicated temporary directory to store the xml result data 1453551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // per run to ensure clean state and make it possible to launch multiple 1463551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // processes in parallel. 1473551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) base::FilePath output_file; 14858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) CHECK(file_util::CreateNewTempDirectory(FilePath::StringType(), 14958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) &output_file)); 15058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) output_file = output_file.AppendASCII("test_results.xml"); 1513551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 1523551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) std::vector<std::string> test_names; 1533551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) for (size_t i = 0; i < tests_.size(); i++) 1543551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) test_names.push_back(tests_[i].GetFullName()); 1553551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 1563551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) CommandLine cmd_line( 1573551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) GetCommandLineForChildGTestProcess(test_names, output_file)); 1583551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 1593551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // Adjust the timeout depending on how many tests we're running 1603551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // (note that e.g. the last batch of tests will be smaller). 1613551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // TODO(phajdan.jr): Consider an adaptive timeout, which can change 1623551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // depending on how many tests ran and how many remain. 1633551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // Note: do NOT parse child's stdout to do that, it's known to be 1643551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // unreliable (e.g. buffering issues can mix up the output). 1653551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) base::TimeDelta timeout = 16668043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) test_names.size() * TestTimeouts::test_launcher_timeout(); 1673551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 16858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) parallel_launcher_.LaunchChildGTestProcess( 16958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) cmd_line, 17058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) std::string(), 17158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) timeout, 17258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) Bind(&UnitTestLauncherDelegate::GTestCallback, 17358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) base::Unretained(this), 17458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) tests_, 17558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) output_file)); 17658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) tests_.clear(); 17758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) } 17858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 17958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) void GTestCallback(const std::vector<TestLaunchInfo>& tests, 18058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) const FilePath& output_file, 18158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) int exit_code, 18268043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) const TimeDelta& elapsed_time, 18358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) bool was_timeout, 18458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) const std::string& output) { 18558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) DCHECK(thread_checker_.CalledOnValidThread()); 18658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) std::vector<TestLaunchInfo> tests_to_relaunch_after_interruption; 18758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) bool called_any_callbacks = 18858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) ProcessTestResults(tests, 18958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) output_file, 19058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) output, 19158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) exit_code, 19258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) was_timeout, 19358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) &tests_to_relaunch_after_interruption); 19458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 19558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) for (size_t i = 0; i < tests_to_relaunch_after_interruption.size(); i++) 19658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) tests_.push_back(tests_to_relaunch_after_interruption[i]); 19758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) RunRemainingTests(); 19858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 19958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) if (called_any_callbacks) 20058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) parallel_launcher_.ResetOutputWatchdog(); 20158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 20258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) // The temporary file's directory is also temporary. 20358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) DeleteFile(output_file.DirName(), true); 20458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) } 20558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 20658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) static bool ProcessTestResults( 20758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) const std::vector<TestLaunchInfo>& tests, 208424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) const base::FilePath& output_file, 20958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) const std::string& output, 210424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) int exit_code, 21158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) bool was_timeout, 21258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) std::vector<TestLaunchInfo>* tests_to_relaunch_after_interruption) { 2133551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) std::vector<TestResult> test_results; 214424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) bool crashed = false; 215424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) bool have_test_results = 216424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) ProcessGTestOutput(output_file, &test_results, &crashed); 2173551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 21858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) bool called_any_callback = false; 21958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 2203551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) if (have_test_results) { 2213551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // TODO(phajdan.jr): Check for duplicates and mismatches between 2223551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // the results we got from XML file and tests we intended to run. 223424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) std::map<std::string, TestResult> results_map; 2243551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) for (size_t i = 0; i < test_results.size(); i++) 225424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) results_map[test_results[i].GetFullName()] = test_results[i]; 226424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) 22758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) bool had_interrupted_test = false; 22858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 22958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) for (size_t i = 0; i < tests.size(); i++) { 23058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) if (ContainsKey(results_map, tests[i].GetFullName())) { 23158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) TestResult test_result = results_map[tests[i].GetFullName()]; 23258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) if (test_result.status == TestResult::TEST_CRASH) { 23358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) had_interrupted_test = true; 23458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 23558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) if (was_timeout) { 23658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) // Fix up the test status: we forcibly kill the child process 23758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) // after the timeout, so from XML results it looks just like 23858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) // a crash. 23958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) test_result.status = TestResult::TEST_TIMEOUT; 24058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) } 24168043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) } else if (test_result.status == TestResult::TEST_SUCCESS || 24268043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) test_result.status == TestResult::TEST_FAILURE) { 24368043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) // We run multiple tests in a batch with a timeout applied 24468043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) // to the entire batch. It is possible that with other tests 24568043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) // running quickly some tests take longer than the per-test timeout. 24668043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) // For consistent handling of tests independent of order and other 24768043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) // factors, mark them as timing out. 24868043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) if (test_result.elapsed_time > 24968043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) TestTimeouts::test_launcher_timeout()) { 25068043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) test_result.status = TestResult::TEST_TIMEOUT; 25168043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) } 25258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) } 2534e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) test_result.output_snippet = 2544e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) GetTestOutputSnippet(test_result, output); 25558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) tests[i].callback.Run(test_result); 25658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) called_any_callback = true; 25758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) } else if (had_interrupted_test) { 25858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) tests_to_relaunch_after_interruption->push_back(tests[i]); 259424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) } else { 260424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) // TODO(phajdan.jr): Explicitly pass the info that the test didn't 261424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) // run for a mysterious reason. 26258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) LOG(ERROR) << "no test result for " << tests[i].GetFullName(); 263424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) TestResult test_result; 26458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) test_result.test_case_name = tests[i].test_case_name; 26558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) test_result.test_name = tests[i].test_name; 26658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) test_result.status = TestResult::TEST_UNKNOWN; 2674e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) test_result.output_snippet = 2684e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) GetTestOutputSnippet(test_result, output); 26958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) tests[i].callback.Run(test_result); 27058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) called_any_callback = true; 271424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) } 2723551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) } 2733551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 274424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) // TODO(phajdan.jr): Handle the case where processing XML output 275424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) // indicates a crash but none of the test results is marked as crashing. 276424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) 2773551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // TODO(phajdan.jr): Handle the case where the exit code is non-zero 2783551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // but results file indicates that all tests passed (e.g. crash during 2793551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // shutdown). 2803551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) } else { 28158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) fprintf(stdout, 28258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) "Failed to get out-of-band test success data, " 28358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) "dumping full stdio below:\n%s\n", 28458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) output.c_str()); 28558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) fflush(stdout); 28658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 2873551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // We do not have reliable details about test results (parsing test 2883551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // stdout is known to be unreliable), apply the executable exit code 2893551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // to all tests. 2903551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // TODO(phajdan.jr): Be smarter about this, e.g. retry each test 2913551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) // individually. 29258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) for (size_t i = 0; i < tests.size(); i++) { 2933551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) TestResult test_result; 29458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) test_result.test_case_name = tests[i].test_case_name; 29558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) test_result.test_name = tests[i].test_name; 29658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) test_result.status = TestResult::TEST_UNKNOWN; 29758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) tests[i].callback.Run(test_result); 29858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) called_any_callback = true; 2993551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) } 3003551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) } 30158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 30258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) return called_any_callback; 3033551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) } 3043551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 30568043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) ThreadChecker thread_checker_; 30668043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) 30758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) ParallelTestLauncher parallel_launcher_; 30858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 30958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) // Maximum number of tests to run in a single batch. 31058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) size_t batch_limit_; 31158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 3123551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) std::vector<TestLaunchInfo> tests_; 3133551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)}; 3143551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 31558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)bool GetSwitchValueAsInt(const std::string& switch_name, int* result) { 31658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) if (!CommandLine::ForCurrentProcess()->HasSwitch(switch_name)) 31758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) return true; 31858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 31958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) std::string switch_value = 32058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) CommandLine::ForCurrentProcess()->GetSwitchValueASCII(switch_name); 32158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) if (!StringToInt(switch_value, result) || *result < 1) { 32258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) LOG(ERROR) << "Invalid value for " << switch_name << ": " << switch_value; 32358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) return false; 32458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) } 32558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 32658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) return true; 32758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)} 32858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 3293551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)} // namespace 3303551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 3313551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)int LaunchUnitTests(int argc, 3323551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) char** argv, 3333551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) const RunTestSuiteCallback& run_test_suite) { 3343551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) CommandLine::Init(argc, argv); 335d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) if (CommandLine::ForCurrentProcess()->HasSwitch(kGTestHelpFlag) || 336d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) CommandLine::ForCurrentProcess()->HasSwitch(kSingleProcessTestsFlag) || 3373551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) !CommandLine::ForCurrentProcess()->HasSwitch(kBraveNewTestLauncherFlag)) { 3383551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return run_test_suite.Run(); 3393551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) } 3403551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 341d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) if (CommandLine::ForCurrentProcess()->HasSwitch(kHelpFlag)) { 342d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) PrintUsage(); 343d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) return 0; 344d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) } 345d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) 3463551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) base::TimeTicks start_time(base::TimeTicks::Now()); 3473551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 34858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) testing::InitGoogleTest(&argc, argv); 34958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) TestTimeouts::Initialize(); 35058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 35158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) int jobs = SysInfo::NumberOfProcessors(); 35258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) if (!GetSwitchValueAsInt(switches::kTestLauncherJobs, &jobs)) 35358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) return 1; 35458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 35558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) int batch_limit = kDefaultTestBatchLimit; 35658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) if (!GetSwitchValueAsInt(switches::kTestLauncherBatchLimit, &batch_limit)) 35758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) return 1; 35858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 3593551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) fprintf(stdout, 36058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) "Starting tests (using %d parallel jobs)...\n" 36158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) "IMPORTANT DEBUGGING NOTE: batches of tests are run inside their\n" 36258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) "own process. For debugging a test inside a debugger, use the\n" 36358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) "--gtest_filter=<your_test_name> flag along with\n" 36458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) "--single-process-tests.\n", jobs); 3653551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) fflush(stdout); 3663551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 36758537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) MessageLoopForIO message_loop; 3683551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 36958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) base::UnitTestLauncherDelegate delegate(jobs, batch_limit); 3703551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) int exit_code = base::LaunchTests(&delegate, argc, argv); 3713551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 3723551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) fprintf(stdout, 3733551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) "Tests took %" PRId64 " seconds.\n", 3743551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) (base::TimeTicks::Now() - start_time).InSeconds()); 3753551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) fflush(stdout); 3763551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 3773551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) return exit_code; 3783551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)} 3793551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 380424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles)} // namespace base 381