1#!/usr/bin/env python
2#
3# Copyright 2005, Google Inc.
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions are
8# met:
9#
10#     * Redistributions of source code must retain the above copyright
11# notice, this list of conditions and the following disclaimer.
12#     * Redistributions in binary form must reproduce the above
13# copyright notice, this list of conditions and the following disclaimer
14# in the documentation and/or other materials provided with the
15# distribution.
16#     * Neither the name of Google Inc. nor the names of its
17# contributors may be used to endorse or promote products derived from
18# this software without specific prior written permission.
19#
20# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
21# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
22# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
23# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
24# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
25# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
26# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
27# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
28# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
29# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
30# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
31
32"""Unit test for Google Test test filters.
33
34A user can specify which test(s) in a Google Test program to run via either
35the GTEST_FILTER environment variable or the --gtest_filter flag.
36This script tests such functionality by invoking
37gtest_filter_unittest_ (a program written with Google Test) with different
38environments and command line flags.
39
40Note that test sharding may also influence which tests are filtered. Therefore,
41we test that here also.
42"""
43
44__author__ = 'wan@google.com (Zhanyong Wan)'
45
46import os
47import re
48import sets
49import tempfile
50import unittest
51import gtest_test_utils
52
53# Constants.
54
55# The environment variable for specifying the test filters.
56FILTER_ENV_VAR = 'GTEST_FILTER'
57
58# The environment variables for test sharding.
59TOTAL_SHARDS_ENV_VAR = 'GTEST_TOTAL_SHARDS'
60SHARD_INDEX_ENV_VAR = 'GTEST_SHARD_INDEX'
61SHARD_STATUS_FILE_ENV_VAR = 'GTEST_SHARD_STATUS_FILE'
62
63# The command line flag for specifying the test filters.
64FILTER_FLAG = 'gtest_filter'
65
66# The command line flag for including disabled tests.
67ALSO_RUN_DISABED_TESTS_FLAG = 'gtest_also_run_disabled_tests'
68
69# Command to run the gtest_filter_unittest_ program.
70COMMAND = os.path.join(gtest_test_utils.GetBuildDir(),
71                       'gtest_filter_unittest_')
72
73# Regex for determining whether parameterized tests are enabled in the binary.
74PARAM_TEST_REGEX = re.compile(r'/ParamTest')
75
76# Regex for parsing test case names from Google Test's output.
77TEST_CASE_REGEX = re.compile(r'^\[\-+\] \d+ tests? from (\w+(/\w+)?)')
78
79# Regex for parsing test names from Google Test's output.
80TEST_REGEX = re.compile(r'^\[\s*RUN\s*\].*\.(\w+(/\w+)?)')
81
82# Full names of all tests in gtest_filter_unittests_.
83PARAM_TESTS = [
84    'SeqP/ParamTest.TestX/0',
85    'SeqP/ParamTest.TestX/1',
86    'SeqP/ParamTest.TestY/0',
87    'SeqP/ParamTest.TestY/1',
88    'SeqQ/ParamTest.TestX/0',
89    'SeqQ/ParamTest.TestX/1',
90    'SeqQ/ParamTest.TestY/0',
91    'SeqQ/ParamTest.TestY/1',
92    ]
93
94DISABLED_TESTS = [
95    'BarTest.DISABLED_TestFour',
96    'BarTest.DISABLED_TestFive',
97    'BazTest.DISABLED_TestC',
98    'DISABLED_FoobarTest.Test1',
99    'DISABLED_FoobarTest.DISABLED_Test2',
100    'DISABLED_FoobarbazTest.TestA',
101    ]
102
103# All the non-disabled tests.
104ACTIVE_TESTS = [
105    'FooTest.Abc',
106    'FooTest.Xyz',
107
108    'BarTest.TestOne',
109    'BarTest.TestTwo',
110    'BarTest.TestThree',
111
112    'BazTest.TestOne',
113    'BazTest.TestA',
114    'BazTest.TestB',
115
116    'HasDeathTest.Test1',
117    'HasDeathTest.Test2',
118    ] + PARAM_TESTS
119
120param_tests_present = None
121
122# Utilities.
123
124
125def SetEnvVar(env_var, value):
126  """Sets the env variable to 'value'; unsets it when 'value' is None."""
127
128  if value is not None:
129    os.environ[env_var] = value
130  elif env_var in os.environ:
131    del os.environ[env_var]
132
133
134def Run(command):
135  """Runs a Google Test program and returns a list of full names of the
136  tests that were run along with the test exit code.
137  """
138
139  stdout_file = os.popen(command, 'r')
140  tests_run = []
141  test_case = ''
142  test = ''
143  for line in stdout_file:
144    match = TEST_CASE_REGEX.match(line)
145    if match is not None:
146      test_case = match.group(1)
147    else:
148      match = TEST_REGEX.match(line)
149      if match is not None:
150        test = match.group(1)
151        tests_run += [test_case + '.' + test]
152  exit_code = stdout_file.close()
153  return (tests_run, exit_code)
154
155
156def InvokeWithModifiedEnv(extra_env, function, *args, **kwargs):
157  """Runs the given function and arguments in a modified environment."""
158  try:
159    original_env = os.environ.copy()
160    os.environ.update(extra_env)
161    return function(*args, **kwargs)
162  finally:
163    for key in extra_env.iterkeys():
164      if key in original_env:
165        os.environ[key] = original_env[key]
166      else:
167        del os.environ[key]
168
169
170def RunWithSharding(total_shards, shard_index, command):
171  """Runs the Google Test program shard and returns a list of full names of the
172  tests that were run along with the exit code.
173  """
174
175  extra_env = {SHARD_INDEX_ENV_VAR: str(shard_index),
176               TOTAL_SHARDS_ENV_VAR: str(total_shards)}
177  return InvokeWithModifiedEnv(extra_env, Run, command)
178
179# The unit test.
180
181
182class GTestFilterUnitTest(unittest.TestCase):
183  """Tests using the GTEST_FILTER environment variable or the
184  --gtest_filter flag to filter tests.
185  """
186
187  # Utilities.
188
189  def AssertSetEqual(self, lhs, rhs):
190    """Asserts that two sets are equal."""
191
192    for elem in lhs:
193      self.assert_(elem in rhs, '%s in %s' % (elem, rhs))
194
195    for elem in rhs:
196      self.assert_(elem in lhs, '%s in %s' % (elem, lhs))
197
198  def AssertPartitionIsValid(self, set_var, list_of_sets):
199    """Asserts that list_of_sets is a valid partition of set_var."""
200
201    full_partition = []
202    for slice_var in list_of_sets:
203      full_partition.extend(slice_var)
204    self.assertEqual(len(set_var), len(full_partition))
205    self.assertEqual(sets.Set(set_var), sets.Set(full_partition))
206
207  def RunAndVerify(self, gtest_filter, tests_to_run):
208    """Runs gtest_flag_unittest_ with the given filter, and verifies
209    that the right set of tests were run.
210    """
211    # Adjust tests_to_run in case value parameterized tests are disabled
212    # in the binary.
213    global param_tests_present
214    if not param_tests_present:
215      tests_to_run = list(sets.Set(tests_to_run) - sets.Set(PARAM_TESTS))
216
217    # First, tests using GTEST_FILTER.
218
219    SetEnvVar(FILTER_ENV_VAR, gtest_filter)
220    tests_run = Run(COMMAND)[0]
221    SetEnvVar(FILTER_ENV_VAR, None)
222
223    self.AssertSetEqual(tests_run, tests_to_run)
224
225    # Next, tests using --gtest_filter.
226
227    if gtest_filter is None:
228      command = COMMAND
229    else:
230      command = '%s --%s=%s' % (COMMAND, FILTER_FLAG, gtest_filter)
231
232    tests_run = Run(command)[0]
233    self.AssertSetEqual(tests_run, tests_to_run)
234
235  def RunAndVerifyWithSharding(self, gtest_filter, total_shards, tests_to_run,
236                               command=COMMAND, check_exit_0=False):
237    """Runs all shards of gtest_flag_unittest_ with the given filter, and
238    verifies that the right set of tests were run. The union of tests run
239    on each shard should be identical to tests_to_run, without duplicates.
240    If check_exit_0, make sure that all shards returned 0.
241    """
242    SetEnvVar(FILTER_ENV_VAR, gtest_filter)
243    partition = []
244    for i in range(0, total_shards):
245      (tests_run, exit_code) = RunWithSharding(total_shards, i, command)
246      if check_exit_0:
247        self.assert_(exit_code is None)
248      partition.append(tests_run)
249
250    self.AssertPartitionIsValid(tests_to_run, partition)
251    SetEnvVar(FILTER_ENV_VAR, None)
252
253  def RunAndVerifyAllowingDisabled(self, gtest_filter, tests_to_run):
254    """Runs gtest_flag_unittest_ with the given filter, and enables
255    disabled tests. Verifies that the right set of tests were run.
256    """
257    # Construct the command line.
258    command = '%s --%s' % (COMMAND, ALSO_RUN_DISABED_TESTS_FLAG)
259    if gtest_filter is not None:
260      command = '%s --%s=%s' % (command, FILTER_FLAG, gtest_filter)
261
262    tests_run = Run(command)[0]
263    self.AssertSetEqual(tests_run, tests_to_run)
264
265  def setUp(self):
266    """Sets up test case. Determines whether value-parameterized tests are
267    enabled in the binary and sets flags accordingly.
268    """
269    global param_tests_present
270    if param_tests_present is None:
271      param_tests_present = PARAM_TEST_REGEX.search(
272          '\n'.join(os.popen(COMMAND, 'r').readlines())) is not None
273
274  def testDefaultBehavior(self):
275    """Tests the behavior of not specifying the filter."""
276
277    self.RunAndVerify(None, ACTIVE_TESTS)
278
279  def testDefaultBehaviorWithShards(self):
280    """Tests the behavior of not specifying the filter, with sharding
281    enabled.
282    """
283    self.RunAndVerifyWithSharding(None, 1, ACTIVE_TESTS)
284    self.RunAndVerifyWithSharding(None, 2, ACTIVE_TESTS)
285    self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS) - 1, ACTIVE_TESTS)
286    self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS), ACTIVE_TESTS)
287    self.RunAndVerifyWithSharding(None, len(ACTIVE_TESTS) + 1, ACTIVE_TESTS)
288
289  def testEmptyFilter(self):
290    """Tests an empty filter."""
291
292    self.RunAndVerify('', [])
293    self.RunAndVerifyWithSharding('', 1, [])
294    self.RunAndVerifyWithSharding('', 2, [])
295
296  def testBadFilter(self):
297    """Tests a filter that matches nothing."""
298
299    self.RunAndVerify('BadFilter', [])
300    self.RunAndVerifyAllowingDisabled('BadFilter', [])
301
302  def testFullName(self):
303    """Tests filtering by full name."""
304
305    self.RunAndVerify('FooTest.Xyz', ['FooTest.Xyz'])
306    self.RunAndVerifyAllowingDisabled('FooTest.Xyz', ['FooTest.Xyz'])
307    self.RunAndVerifyWithSharding('FooTest.Xyz', 5, ['FooTest.Xyz'])
308
309  def testUniversalFilters(self):
310    """Tests filters that match everything."""
311
312    self.RunAndVerify('*', ACTIVE_TESTS)
313    self.RunAndVerify('*.*', ACTIVE_TESTS)
314    self.RunAndVerifyWithSharding('*.*', len(ACTIVE_TESTS) - 3, ACTIVE_TESTS)
315    self.RunAndVerifyAllowingDisabled('*', ACTIVE_TESTS + DISABLED_TESTS)
316    self.RunAndVerifyAllowingDisabled('*.*', ACTIVE_TESTS + DISABLED_TESTS)
317
318  def testFilterByTestCase(self):
319    """Tests filtering by test case name."""
320
321    self.RunAndVerify('FooTest.*', ['FooTest.Abc', 'FooTest.Xyz'])
322
323    BAZ_TESTS = ['BazTest.TestOne', 'BazTest.TestA', 'BazTest.TestB']
324    self.RunAndVerify('BazTest.*', BAZ_TESTS)
325    self.RunAndVerifyAllowingDisabled('BazTest.*',
326                                      BAZ_TESTS + ['BazTest.DISABLED_TestC'])
327
328  def testFilterByTest(self):
329    """Tests filtering by test name."""
330
331    self.RunAndVerify('*.TestOne', ['BarTest.TestOne', 'BazTest.TestOne'])
332
333  def testFilterDisabledTests(self):
334    """Select only the disabled tests to run."""
335
336    self.RunAndVerify('DISABLED_FoobarTest.Test1', [])
337    self.RunAndVerifyAllowingDisabled('DISABLED_FoobarTest.Test1',
338                                      ['DISABLED_FoobarTest.Test1'])
339
340    self.RunAndVerify('*DISABLED_*', [])
341    self.RunAndVerifyAllowingDisabled('*DISABLED_*', DISABLED_TESTS)
342
343    self.RunAndVerify('*.DISABLED_*', [])
344    self.RunAndVerifyAllowingDisabled('*.DISABLED_*', [
345        'BarTest.DISABLED_TestFour',
346        'BarTest.DISABLED_TestFive',
347        'BazTest.DISABLED_TestC',
348        'DISABLED_FoobarTest.DISABLED_Test2',
349        ])
350
351    self.RunAndVerify('DISABLED_*', [])
352    self.RunAndVerifyAllowingDisabled('DISABLED_*', [
353        'DISABLED_FoobarTest.Test1',
354        'DISABLED_FoobarTest.DISABLED_Test2',
355        'DISABLED_FoobarbazTest.TestA',
356        ])
357
358  def testWildcardInTestCaseName(self):
359    """Tests using wildcard in the test case name."""
360
361    self.RunAndVerify('*a*.*', [
362        'BarTest.TestOne',
363        'BarTest.TestTwo',
364        'BarTest.TestThree',
365
366        'BazTest.TestOne',
367        'BazTest.TestA',
368        'BazTest.TestB',
369
370        'HasDeathTest.Test1',
371        'HasDeathTest.Test2', ] + PARAM_TESTS)
372
373  def testWildcardInTestName(self):
374    """Tests using wildcard in the test name."""
375
376    self.RunAndVerify('*.*A*', ['FooTest.Abc', 'BazTest.TestA'])
377
378  def testFilterWithoutDot(self):
379    """Tests a filter that has no '.' in it."""
380
381    self.RunAndVerify('*z*', [
382        'FooTest.Xyz',
383
384        'BazTest.TestOne',
385        'BazTest.TestA',
386        'BazTest.TestB',
387        ])
388
389  def testTwoPatterns(self):
390    """Tests filters that consist of two patterns."""
391
392    self.RunAndVerify('Foo*.*:*A*', [
393        'FooTest.Abc',
394        'FooTest.Xyz',
395
396        'BazTest.TestA',
397        ])
398
399    # An empty pattern + a non-empty one
400    self.RunAndVerify(':*A*', ['FooTest.Abc', 'BazTest.TestA'])
401
402  def testThreePatterns(self):
403    """Tests filters that consist of three patterns."""
404
405    self.RunAndVerify('*oo*:*A*:*One', [
406        'FooTest.Abc',
407        'FooTest.Xyz',
408
409        'BarTest.TestOne',
410
411        'BazTest.TestOne',
412        'BazTest.TestA',
413        ])
414
415    # The 2nd pattern is empty.
416    self.RunAndVerify('*oo*::*One', [
417        'FooTest.Abc',
418        'FooTest.Xyz',
419
420        'BarTest.TestOne',
421
422        'BazTest.TestOne',
423        ])
424
425    # The last 2 patterns are empty.
426    self.RunAndVerify('*oo*::', [
427        'FooTest.Abc',
428        'FooTest.Xyz',
429        ])
430
431  def testNegativeFilters(self):
432    self.RunAndVerify('*-HasDeathTest.Test1', [
433        'FooTest.Abc',
434        'FooTest.Xyz',
435
436        'BarTest.TestOne',
437        'BarTest.TestTwo',
438        'BarTest.TestThree',
439
440        'BazTest.TestOne',
441        'BazTest.TestA',
442        'BazTest.TestB',
443
444        'HasDeathTest.Test2',
445        ] + PARAM_TESTS)
446
447    self.RunAndVerify('*-FooTest.Abc:HasDeathTest.*', [
448        'FooTest.Xyz',
449
450        'BarTest.TestOne',
451        'BarTest.TestTwo',
452        'BarTest.TestThree',
453
454        'BazTest.TestOne',
455        'BazTest.TestA',
456        'BazTest.TestB',
457        ] + PARAM_TESTS)
458
459    self.RunAndVerify('BarTest.*-BarTest.TestOne', [
460        'BarTest.TestTwo',
461        'BarTest.TestThree',
462        ])
463
464    # Tests without leading '*'.
465    self.RunAndVerify('-FooTest.Abc:FooTest.Xyz:HasDeathTest.*', [
466        'BarTest.TestOne',
467        'BarTest.TestTwo',
468        'BarTest.TestThree',
469
470        'BazTest.TestOne',
471        'BazTest.TestA',
472        'BazTest.TestB',
473        ] + PARAM_TESTS)
474
475    # Value parameterized tests.
476    self.RunAndVerify('*/*', PARAM_TESTS)
477
478    # Value parameterized tests filtering by the sequence name.
479    self.RunAndVerify('SeqP/*', [
480        'SeqP/ParamTest.TestX/0',
481        'SeqP/ParamTest.TestX/1',
482        'SeqP/ParamTest.TestY/0',
483        'SeqP/ParamTest.TestY/1',
484        ])
485
486    # Value parameterized tests filtering by the test name.
487    self.RunAndVerify('*/0', [
488        'SeqP/ParamTest.TestX/0',
489        'SeqP/ParamTest.TestY/0',
490        'SeqQ/ParamTest.TestX/0',
491        'SeqQ/ParamTest.TestY/0',
492        ])
493
494  def testFlagOverridesEnvVar(self):
495    """Tests that the --gtest_filter flag overrides the GTEST_FILTER
496    environment variable.
497    """
498
499    SetEnvVar(FILTER_ENV_VAR, 'Foo*')
500    command = '%s --%s=%s' % (COMMAND, FILTER_FLAG, '*One')
501    tests_run = Run(command)[0]
502    SetEnvVar(FILTER_ENV_VAR, None)
503
504    self.AssertSetEqual(tests_run, ['BarTest.TestOne', 'BazTest.TestOne'])
505
506  def testShardStatusFileIsCreated(self):
507    """Tests that the shard file is created if specified in the environment."""
508
509    test_tmpdir = tempfile.mkdtemp()
510    shard_status_file = os.path.join(test_tmpdir, 'shard_status_file')
511    self.assert_(not os.path.exists(shard_status_file))
512
513    extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file}
514    stdout_file = InvokeWithModifiedEnv(extra_env, os.popen, COMMAND, 'r')
515    try:
516      stdout_file.readlines()
517    finally:
518      stdout_file.close()
519      self.assert_(os.path.exists(shard_status_file))
520      os.remove(shard_status_file)
521      os.removedirs(test_tmpdir)
522
523  def testShardStatusFileIsCreatedWithListTests(self):
524    """Tests that the shard file is created with --gtest_list_tests."""
525
526    test_tmpdir = tempfile.mkdtemp()
527    shard_status_file = os.path.join(test_tmpdir, 'shard_status_file2')
528    self.assert_(not os.path.exists(shard_status_file))
529
530    extra_env = {SHARD_STATUS_FILE_ENV_VAR: shard_status_file}
531    stdout_file = InvokeWithModifiedEnv(extra_env, os.popen,
532                                        '%s --gtest_list_tests' % COMMAND, 'r')
533    try:
534      stdout_file.readlines()
535    finally:
536      stdout_file.close()
537      self.assert_(os.path.exists(shard_status_file))
538      os.remove(shard_status_file)
539      os.removedirs(test_tmpdir)
540
541  def testShardingWorksWithDeathTests(self):
542    """Tests integration with death tests and sharding."""
543    gtest_filter = 'HasDeathTest.*:SeqP/*'
544    expected_tests = [
545        'HasDeathTest.Test1',
546        'HasDeathTest.Test2',
547
548        'SeqP/ParamTest.TestX/0',
549        'SeqP/ParamTest.TestX/1',
550        'SeqP/ParamTest.TestY/0',
551        'SeqP/ParamTest.TestY/1',
552        ]
553
554    for command in (COMMAND + ' --gtest_death_test_style=threadsafe',
555                    COMMAND + ' --gtest_death_test_style=fast'):
556      self.RunAndVerifyWithSharding(gtest_filter, 3, expected_tests,
557                                    check_exit_0=True, command=command)
558      self.RunAndVerifyWithSharding(gtest_filter, 5, expected_tests,
559                                    check_exit_0=True, command=command)
560
561if __name__ == '__main__':
562  gtest_test_utils.Main()
563