1f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan#!/usr/bin/env python 2f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# 3f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# Copyright 2009 Google Inc. All Rights Reserved. 4f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# 5f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# Redistribution and use in source and binary forms, with or without 6f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# modification, are permitted provided that the following conditions are 7f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# met: 8f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# 9f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# * Redistributions of source code must retain the above copyright 10f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# notice, this list of conditions and the following disclaimer. 11f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# * Redistributions in binary form must reproduce the above 12f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# copyright notice, this list of conditions and the following disclaimer 13f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# in the documentation and/or other materials provided with the 14f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# distribution. 15f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# * Neither the name of Google Inc. nor the names of its 16f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# contributors may be used to endorse or promote products derived from 17f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# this software without specific prior written permission. 18f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# 19f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 20f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 21f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 22f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 23f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 24f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 25f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 26f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 27f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 28f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 31f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan"""Verifies that test shuffling works.""" 32f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 33f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan__author__ = 'wan@google.com (Zhanyong Wan)' 34f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 35f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wanimport os 36f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wanimport gtest_test_utils 37f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 38f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# Command to run the gtest_shuffle_test_ program. 39f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wanCOMMAND = gtest_test_utils.GetTestExecutablePath('gtest_shuffle_test_') 40f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 41f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan# The environment variables for test sharding. 42f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wanTOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS' 43f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wanSHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX' 44f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 45f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wanTEST_FILTER = 'A*.A:A*.B:C*' 46f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 47f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wanALL_TESTS = [] 48f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wanACTIVE_TESTS = [] 49f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wanFILTERED_TESTS = [] 50f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wanSHARDED_TESTS = [] 51f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 52f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wanSHUFFLED_ALL_TESTS = [] 53f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wanSHUFFLED_ACTIVE_TESTS = [] 54f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wanSHUFFLED_FILTERED_TESTS = [] 55f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wanSHUFFLED_SHARDED_TESTS = [] 56f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 57f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 58f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wandef AlsoRunDisabledTestsFlag(): 59f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan return '--gtest_also_run_disabled_tests' 60f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 61f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 62f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wandef FilterFlag(test_filter): 63f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan return '--gtest_filter=%s' % (test_filter,) 64f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 65f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 66f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wandef RepeatFlag(n): 67f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan return '--gtest_repeat=%s' % (n,) 68f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 69f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 70f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wandef ShuffleFlag(): 71f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan return '--gtest_shuffle' 72f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 73f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 74f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wandef RandomSeedFlag(n): 75f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan return '--gtest_random_seed=%s' % (n,) 76f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 77f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 78f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wandef RunAndReturnOutput(extra_env, args): 79f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan """Runs the test program and returns its output.""" 80f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 81e7dfd7ed918c9714f096c873544ed11e91ff8106vladlosev environ_copy = os.environ.copy() 82e7dfd7ed918c9714f096c873544ed11e91ff8106vladlosev environ_copy.update(extra_env) 83e7dfd7ed918c9714f096c873544ed11e91ff8106vladlosev 84df246651195747a948cad465dc717133e22973d6vladlosev return gtest_test_utils.Subprocess([COMMAND] + args, env=environ_copy).output 85f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 86f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 87f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wandef GetTestsForAllIterations(extra_env, args): 88f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan """Runs the test program and returns a list of test lists. 89f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 90f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan Args: 91f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan extra_env: a map from environment variables to their values 92f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan args: command line flags to pass to gtest_shuffle_test_ 93f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 94f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan Returns: 95f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan A list where the i-th element is the list of tests run in the i-th 96f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan test iteration. 97f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan """ 98f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 99f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan test_iterations = [] 100f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan for line in RunAndReturnOutput(extra_env, args).split('\n'): 101f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan if line.startswith('----'): 102f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan tests = [] 103f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan test_iterations.append(tests) 104f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan elif line.strip(): 105f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan tests.append(line.strip()) # 'TestCaseName.TestName' 106f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 107f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan return test_iterations 108f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 109f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 110f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wandef GetTestCases(tests): 111f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan """Returns a list of test cases in the given full test names. 112f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 113f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan Args: 114f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan tests: a list of full test names 115f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 116f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan Returns: 117f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan A list of test cases from 'tests', in their original order. 118f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan Consecutive duplicates are removed. 119f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan """ 120f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 121f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan test_cases = [] 122f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan for test in tests: 123f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan test_case = test.split('.')[0] 124f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan if not test_case in test_cases: 125f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan test_cases.append(test_case) 126f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 127f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan return test_cases 128f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 129f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 130f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wandef CalculateTestLists(): 131f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan """Calculates the list of tests run under different flags.""" 132f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 133f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan if not ALL_TESTS: 134f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan ALL_TESTS.extend( 135f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan GetTestsForAllIterations({}, [AlsoRunDisabledTestsFlag()])[0]) 136f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 137f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan if not ACTIVE_TESTS: 138f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan ACTIVE_TESTS.extend(GetTestsForAllIterations({}, [])[0]) 139f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 140f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan if not FILTERED_TESTS: 141f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan FILTERED_TESTS.extend( 142f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan GetTestsForAllIterations({}, [FilterFlag(TEST_FILTER)])[0]) 143f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 144f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan if not SHARDED_TESTS: 145f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan SHARDED_TESTS.extend( 146f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 147f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan SHARD_INDEX_ENV_VAR: '1'}, 148f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan [])[0]) 149f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 150f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan if not SHUFFLED_ALL_TESTS: 151f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan SHUFFLED_ALL_TESTS.extend(GetTestsForAllIterations( 152f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan {}, [AlsoRunDisabledTestsFlag(), ShuffleFlag(), RandomSeedFlag(1)])[0]) 153f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 154f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan if not SHUFFLED_ACTIVE_TESTS: 155f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan SHUFFLED_ACTIVE_TESTS.extend(GetTestsForAllIterations( 156f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan {}, [ShuffleFlag(), RandomSeedFlag(1)])[0]) 157f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 158f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan if not SHUFFLED_FILTERED_TESTS: 159f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan SHUFFLED_FILTERED_TESTS.extend(GetTestsForAllIterations( 160f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan {}, [ShuffleFlag(), RandomSeedFlag(1), FilterFlag(TEST_FILTER)])[0]) 161f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 162f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan if not SHUFFLED_SHARDED_TESTS: 163f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan SHUFFLED_SHARDED_TESTS.extend( 164f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 165f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan SHARD_INDEX_ENV_VAR: '1'}, 166f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan [ShuffleFlag(), RandomSeedFlag(1)])[0]) 167f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 168f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 169f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wanclass GTestShuffleUnitTest(gtest_test_utils.TestCase): 170f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan """Tests test shuffling.""" 171f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 172f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan def setUp(self): 173f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan CalculateTestLists() 174f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 175f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan def testShufflePreservesNumberOfTests(self): 176f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assertEqual(len(ALL_TESTS), len(SHUFFLED_ALL_TESTS)) 177f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assertEqual(len(ACTIVE_TESTS), len(SHUFFLED_ACTIVE_TESTS)) 178f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assertEqual(len(FILTERED_TESTS), len(SHUFFLED_FILTERED_TESTS)) 179f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assertEqual(len(SHARDED_TESTS), len(SHUFFLED_SHARDED_TESTS)) 180f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 181f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan def testShuffleChangesTestOrder(self): 182f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assert_(SHUFFLED_ALL_TESTS != ALL_TESTS, SHUFFLED_ALL_TESTS) 183f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assert_(SHUFFLED_ACTIVE_TESTS != ACTIVE_TESTS, SHUFFLED_ACTIVE_TESTS) 184f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assert_(SHUFFLED_FILTERED_TESTS != FILTERED_TESTS, 185f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan SHUFFLED_FILTERED_TESTS) 186f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assert_(SHUFFLED_SHARDED_TESTS != SHARDED_TESTS, 187f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan SHUFFLED_SHARDED_TESTS) 188f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 189f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan def testShuffleChangesTestCaseOrder(self): 190f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assert_(GetTestCases(SHUFFLED_ALL_TESTS) != GetTestCases(ALL_TESTS), 191f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan GetTestCases(SHUFFLED_ALL_TESTS)) 192f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assert_( 193f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan GetTestCases(SHUFFLED_ACTIVE_TESTS) != GetTestCases(ACTIVE_TESTS), 194f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan GetTestCases(SHUFFLED_ACTIVE_TESTS)) 195f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assert_( 196f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan GetTestCases(SHUFFLED_FILTERED_TESTS) != GetTestCases(FILTERED_TESTS), 197f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan GetTestCases(SHUFFLED_FILTERED_TESTS)) 198f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assert_( 199f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan GetTestCases(SHUFFLED_SHARDED_TESTS) != GetTestCases(SHARDED_TESTS), 200f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan GetTestCases(SHUFFLED_SHARDED_TESTS)) 201f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 202f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan def testShuffleDoesNotRepeatTest(self): 203f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan for test in SHUFFLED_ALL_TESTS: 204f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assertEqual(1, SHUFFLED_ALL_TESTS.count(test), 205f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan '%s appears more than once' % (test,)) 206f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan for test in SHUFFLED_ACTIVE_TESTS: 207f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assertEqual(1, SHUFFLED_ACTIVE_TESTS.count(test), 208f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan '%s appears more than once' % (test,)) 209f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan for test in SHUFFLED_FILTERED_TESTS: 210f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assertEqual(1, SHUFFLED_FILTERED_TESTS.count(test), 211f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan '%s appears more than once' % (test,)) 212f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan for test in SHUFFLED_SHARDED_TESTS: 213f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assertEqual(1, SHUFFLED_SHARDED_TESTS.count(test), 214f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan '%s appears more than once' % (test,)) 215f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 216f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan def testShuffleDoesNotCreateNewTest(self): 217f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan for test in SHUFFLED_ALL_TESTS: 218f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assert_(test in ALL_TESTS, '%s is an invalid test' % (test,)) 219f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan for test in SHUFFLED_ACTIVE_TESTS: 220f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assert_(test in ACTIVE_TESTS, '%s is an invalid test' % (test,)) 221f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan for test in SHUFFLED_FILTERED_TESTS: 222f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assert_(test in FILTERED_TESTS, '%s is an invalid test' % (test,)) 223f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan for test in SHUFFLED_SHARDED_TESTS: 224f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assert_(test in SHARDED_TESTS, '%s is an invalid test' % (test,)) 225f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 226f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan def testShuffleIncludesAllTests(self): 227f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan for test in ALL_TESTS: 228f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assert_(test in SHUFFLED_ALL_TESTS, '%s is missing' % (test,)) 229f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan for test in ACTIVE_TESTS: 230f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assert_(test in SHUFFLED_ACTIVE_TESTS, '%s is missing' % (test,)) 231f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan for test in FILTERED_TESTS: 232f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assert_(test in SHUFFLED_FILTERED_TESTS, '%s is missing' % (test,)) 233f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan for test in SHARDED_TESTS: 234f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assert_(test in SHUFFLED_SHARDED_TESTS, '%s is missing' % (test,)) 235f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 236f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan def testShuffleLeavesDeathTestsAtFront(self): 237f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan non_death_test_found = False 238f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan for test in SHUFFLED_ACTIVE_TESTS: 239f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan if 'DeathTest.' in test: 240f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assert_(not non_death_test_found, 241f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan '%s appears after a non-death test' % (test,)) 242f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan else: 243f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan non_death_test_found = True 244f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 245f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan def _VerifyTestCasesDoNotInterleave(self, tests): 246f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan test_cases = [] 247f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan for test in tests: 248f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan [test_case, _] = test.split('.') 249f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan if test_cases and test_cases[-1] != test_case: 250f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan test_cases.append(test_case) 251f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assertEqual(1, test_cases.count(test_case), 252f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 'Test case %s is not grouped together in %s' % 253f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan (test_case, tests)) 254f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 255f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan def testShuffleDoesNotInterleaveTestCases(self): 256f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self._VerifyTestCasesDoNotInterleave(SHUFFLED_ALL_TESTS) 257f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self._VerifyTestCasesDoNotInterleave(SHUFFLED_ACTIVE_TESTS) 258f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self._VerifyTestCasesDoNotInterleave(SHUFFLED_FILTERED_TESTS) 259f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self._VerifyTestCasesDoNotInterleave(SHUFFLED_SHARDED_TESTS) 260f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 261f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan def testShuffleRestoresOrderAfterEachIteration(self): 262f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan # Get the test lists in all 3 iterations, using random seed 1, 2, 263f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan # and 3 respectively. Google Test picks a different seed in each 264f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan # iteration, and this test depends on the current implementation 265f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan # picking successive numbers. This dependency is not ideal, but 266f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan # makes the test much easier to write. 267f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( 268f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan GetTestsForAllIterations( 269f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)])) 270f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 271f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan # Make sure running the tests with random seed 1 gets the same 272f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan # order as in iteration 1 above. 273f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan [tests_with_seed1] = GetTestsForAllIterations( 274f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan {}, [ShuffleFlag(), RandomSeedFlag(1)]) 275f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assertEqual(tests_in_iteration1, tests_with_seed1) 276f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 277f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan # Make sure running the tests with random seed 2 gets the same 278f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan # order as in iteration 2 above. Success means that Google Test 279f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan # correctly restores the test order before re-shuffling at the 280f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan # beginning of iteration 2. 281f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan [tests_with_seed2] = GetTestsForAllIterations( 282f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan {}, [ShuffleFlag(), RandomSeedFlag(2)]) 283f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assertEqual(tests_in_iteration2, tests_with_seed2) 284f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 285f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan # Make sure running the tests with random seed 3 gets the same 286f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan # order as in iteration 3 above. Success means that Google Test 287f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan # correctly restores the test order before re-shuffling at the 288f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan # beginning of iteration 3. 289f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan [tests_with_seed3] = GetTestsForAllIterations( 290f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan {}, [ShuffleFlag(), RandomSeedFlag(3)]) 291f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assertEqual(tests_in_iteration3, tests_with_seed3) 292f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 293f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan def testShuffleGeneratesNewOrderInEachIteration(self): 294f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan [tests_in_iteration1, tests_in_iteration2, tests_in_iteration3] = ( 295f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan GetTestsForAllIterations( 296f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan {}, [ShuffleFlag(), RandomSeedFlag(1), RepeatFlag(3)])) 297f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 298f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assert_(tests_in_iteration1 != tests_in_iteration2, 299f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan tests_in_iteration1) 300f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assert_(tests_in_iteration1 != tests_in_iteration3, 301f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan tests_in_iteration1) 302f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assert_(tests_in_iteration2 != tests_in_iteration3, 303f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan tests_in_iteration2) 304f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 305f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan def testShuffleShardedTestsPreservesPartition(self): 306f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan # If we run M tests on N shards, the same M tests should be run in 307f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan # total, regardless of the random seeds used by the shards. 308f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan [tests1] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 309f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan SHARD_INDEX_ENV_VAR: '0'}, 310f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan [ShuffleFlag(), RandomSeedFlag(1)]) 311f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan [tests2] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 312f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan SHARD_INDEX_ENV_VAR: '1'}, 313f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan [ShuffleFlag(), RandomSeedFlag(20)]) 314f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan [tests3] = GetTestsForAllIterations({TOTAL_SHARDS_ENV_VAR: '3', 315f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan SHARD_INDEX_ENV_VAR: '2'}, 316f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan [ShuffleFlag(), RandomSeedFlag(25)]) 317f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan sorted_sharded_tests = tests1 + tests2 + tests3 318f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan sorted_sharded_tests.sort() 319f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan sorted_active_tests = [] 320f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan sorted_active_tests.extend(ACTIVE_TESTS) 321f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan sorted_active_tests.sort() 322f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan self.assertEqual(sorted_active_tests, sorted_sharded_tests) 323f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan 324f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wanif __name__ == '__main__': 325f19450f3ad156e1dea624b4d37e645bef067ab5czhanyong.wan gtest_test_utils.Main() 326