1# Copyright 2016 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import os
6import string
7import sys
8import tempfile
9import unittest
10import json
11
12from telemetry import decorators
13from telemetry import project_config
14from telemetry.core import util
15from telemetry.testing import browser_test_context
16from telemetry.testing import browser_test_runner
17from telemetry.testing import options_for_unittests
18from telemetry.testing import run_browser_tests
19from telemetry.testing import serially_executed_browser_test_case
20
21
22class BrowserTestRunnerTest(unittest.TestCase):
23
24  def _ExtractTestResults(self, test_result):
25    delimiter = test_result['path_delimiter']
26    failures = []
27    successes = []
28    def _IsLeafNode(node):
29      test_dict = node[1]
30      return ('expected' in test_dict and
31              isinstance(test_dict['expected'], basestring))
32    node_queues = []
33    for t in test_result['tests']:
34      node_queues.append((t, test_result['tests'][t]))
35    while node_queues:
36      node = node_queues.pop()
37      full_test_name, test_dict = node
38      if _IsLeafNode(node):
39        if all(res not in test_dict['expected'].split() for res in
40               test_dict['actual'].split()):
41          failures.append(full_test_name)
42        else:
43          successes.append(full_test_name)
44      else:
45        for k in test_dict:
46          node_queues.append(
47            ('%s%s%s' % (full_test_name, delimiter, k),
48             test_dict[k]))
49    return successes, failures
50
51  def baseTest(self, test_filter,
52               failures, successes, test_name='SimpleTest'):
53    config = project_config.ProjectConfig(
54        top_level_dir=os.path.join(util.GetTelemetryDir(), 'examples'),
55        client_configs=[],
56        benchmark_dirs=[
57            os.path.join(util.GetTelemetryDir(), 'examples', 'browser_tests')]
58    )
59    temp_file = tempfile.NamedTemporaryFile(delete=False)
60    temp_file.close()
61    temp_file_name = temp_file.name
62    try:
63      browser_test_runner.Run(
64          config,
65          [test_name,
66           '--write-full-results-to=%s' % temp_file_name,
67           '--test-filter=%s' % test_filter])
68      with open(temp_file_name) as f:
69        test_result = json.load(f)
70
71      actual_successes, actual_failures = self._ExtractTestResults(test_result)
72      self.assertEquals(set(actual_failures), set(failures))
73      self.assertEquals(set(actual_successes), set(successes))
74    finally:
75      os.remove(temp_file_name)
76
77  @decorators.Disabled('chromeos')  # crbug.com/696553
78  def testJsonOutputFormatNegativeFilter(self):
79    self.baseTest(
80      '^(add|multiplier).*',
81      ['browser_tests.simple_numeric_test.SimpleTest.add_1_and_2',
82       'browser_tests.simple_numeric_test.SimpleTest.add_7_and_3',
83       'browser_tests.simple_numeric_test.SimpleTest.multiplier_simple_2'],
84      ['browser_tests.simple_numeric_test.SimpleTest.add_2_and_3',
85       'browser_tests.simple_numeric_test.SimpleTest.multiplier_simple',
86       'browser_tests.simple_numeric_test.SimpleTest.multiplier_simple_3'])
87
88  @decorators.Disabled('chromeos')  # crbug.com/696553
89  def testJsonOutputWhenSetupClassFailed(self):
90    self.baseTest(
91      '.*',
92      ['browser_tests.failed_tests.SetUpClassFailedTest.dummy_test_0',
93       'browser_tests.failed_tests.SetUpClassFailedTest.dummy_test_1',
94       'browser_tests.failed_tests.SetUpClassFailedTest.dummy_test_2'],
95      [], test_name='SetUpClassFailedTest')
96
97  @decorators.Disabled('chromeos')  # crbug.com/696553
98  def testJsonOutputWhenTearDownClassFailed(self):
99    self.baseTest(
100      '.*',
101      ['browser_tests.failed_tests.TearDownClassFailedTest.dummy_test_0',
102       'browser_tests.failed_tests.TearDownClassFailedTest.dummy_test_1',
103       'browser_tests.failed_tests.TearDownClassFailedTest.dummy_test_2'],
104      [], test_name='TearDownClassFailedTest')
105
106  @decorators.Disabled('chromeos')  # crbug.com/696553
107  def testSetUpProcessCalledOnce(self):
108    self.baseTest(
109      '.*',
110      [],
111      ['browser_tests.process_tests.FailIfSetUpProcessCalledTwice.Dummy_0',
112       'browser_tests.process_tests.FailIfSetUpProcessCalledTwice.Dummy_1',
113       'browser_tests.process_tests.FailIfSetUpProcessCalledTwice.Dummy_2'],
114      test_name='FailIfSetUpProcessCalledTwice')
115
116  @decorators.Disabled('chromeos')  # crbug.com/696553
117  def testTearDownProcessCalledOnce(self):
118    self.baseTest(
119      '.*',
120      [],
121      ['browser_tests.process_tests.FailIfTearDownProcessCalledTwice.Dummy_0',
122       'browser_tests.process_tests.FailIfTearDownProcessCalledTwice.Dummy_1',
123       'browser_tests.process_tests.FailIfTearDownProcessCalledTwice.Dummy_2'],
124      test_name='FailIfTearDownProcessCalledTwice')
125
126  @decorators.Disabled('chromeos')  # crbug.com/696553
127  def testJsonOutputFormatPositiveFilter(self):
128    self.baseTest(
129      '(TestSimple|TestException).*',
130      ['browser_tests.simple_numeric_test.SimpleTest.TestException',
131       'browser_tests.simple_numeric_test.SimpleTest.TestSimple'], [])
132
133  @decorators.Disabled('chromeos')  # crbug.com/696553
134  def testExecutingTestsInSortedOrder(self):
135    alphabetical_tests = []
136    prefix = 'browser_tests.simple_numeric_test.SimpleTest.Alphabetical_'
137    for i in xrange(20):
138      alphabetical_tests.append(prefix + str(i))
139    for c in string.uppercase[:26]:
140      alphabetical_tests.append(prefix + c)
141    for c in string.lowercase[:26]:
142      alphabetical_tests.append(prefix + c)
143    alphabetical_tests.sort()
144    self.baseTest(
145        'Alphabetical', [], alphabetical_tests)
146
147  def shardingRangeTestHelper(self, total_shards, num_tests):
148    shard_ranges = []
149    for shard_index in xrange(0, total_shards):
150      shard_ranges.append(run_browser_tests._TestRangeForShard(
151        total_shards, shard_index, num_tests))
152    # Make assertions about ranges
153    num_tests_run = 0
154    for i in xrange(0, len(shard_ranges)):
155      cur_range = shard_ranges[i]
156      if i < num_tests:
157        self.assertGreater(cur_range[1], cur_range[0])
158        num_tests_run += (cur_range[1] - cur_range[0])
159      else:
160        # Not enough tests to go around all of the shards.
161        self.assertEquals(cur_range[0], cur_range[1])
162    # Make assertions about non-overlapping ranges
163    for i in xrange(1, len(shard_ranges)):
164      prev_range = shard_ranges[i - 1]
165      cur_range = shard_ranges[i]
166      self.assertEquals(prev_range[1], cur_range[0])
167    # Assert that we run all of the tests (very important)
168    self.assertEquals(num_tests_run, num_tests)
169
170  def testShardsWithPrimeNumTests(self):
171    for total_shards in xrange(1, 20):
172      # Nice non-prime number
173      self.shardingRangeTestHelper(total_shards, 101)
174
175  def testShardsWithDivisibleNumTests(self):
176    for total_shards in xrange(1, 6):
177      self.shardingRangeTestHelper(total_shards, 8)
178
179  def testShardBoundaryConditions(self):
180    self.shardingRangeTestHelper(1, 0)
181    self.shardingRangeTestHelper(1, 1)
182    self.shardingRangeTestHelper(2, 1)
183
184  def baseShardingTest(self, total_shards, shard_index, failures, successes,
185                       opt_abbr_input_json_file=None,
186                       opt_test_filter='',
187                       opt_filter_tests_after_sharding=False):
188    config = project_config.ProjectConfig(
189        top_level_dir=os.path.join(util.GetTelemetryDir(), 'examples'),
190        client_configs=[],
191        benchmark_dirs=[
192            os.path.join(util.GetTelemetryDir(), 'examples', 'browser_tests')]
193    )
194    temp_file = tempfile.NamedTemporaryFile(delete=False)
195    temp_file.close()
196    temp_file_name = temp_file.name
197    opt_args = []
198    if opt_abbr_input_json_file:
199      opt_args += [
200        '--read-abbreviated-json-results-from=%s' % opt_abbr_input_json_file]
201    if opt_test_filter:
202      opt_args += [
203        '--test-filter=%s' % opt_test_filter]
204    if opt_filter_tests_after_sharding:
205      opt_args += ['--filter-tests-after-sharding']
206    try:
207      browser_test_runner.Run(
208          config,
209          ['SimpleShardingTest',
210           '--write-full-results-to=%s' % temp_file_name,
211           '--total-shards=%d' % total_shards,
212           '--shard-index=%d' % shard_index] + opt_args)
213      with open(temp_file_name) as f:
214        test_result = json.load(f)
215      actual_successes, actual_failures = self._ExtractTestResults(test_result)
216      self.assertEquals(set(actual_failures), set(failures))
217      self.assertEquals(set(actual_successes), set(successes))
218    finally:
219      os.remove(temp_file_name)
220
221  @decorators.Disabled('chromeos')  # crbug.com/696553
222  def testShardedTestRun(self):
223    self.baseShardingTest(3, 0, [], [
224      'browser_tests.simple_sharding_test.SimpleShardingTest.Test1',
225      'browser_tests.simple_sharding_test.SimpleShardingTest.Test2',
226      'browser_tests.simple_sharding_test.SimpleShardingTest.Test3',
227      'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_0',
228      'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_1',
229    ])
230    self.baseShardingTest(3, 1, [], [
231      'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_2',
232      'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_3',
233      'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_4',
234      'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_5',
235    ])
236    self.baseShardingTest(3, 2, [], [
237      'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_6',
238      'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_7',
239      'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_8',
240      'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_9',
241    ])
242
243  def writeMockTestResultsFile(self):
244    mock_test_results = {
245      'passes': [
246        'Test1',
247        'Test2',
248        'Test3',
249        'passing_test_0',
250        'passing_test_1',
251        'passing_test_2',
252        'passing_test_3',
253        'passing_test_4',
254        'passing_test_5',
255        'passing_test_6',
256        'passing_test_7',
257        'passing_test_8',
258        'passing_test_9',
259      ],
260      'failures': [],
261      'valid': True,
262      'times': {
263        'Test1': 3.0,
264        'Test2': 3.0,
265        'Test3': 3.0,
266        'passing_test_0': 3.0,
267        'passing_test_1': 2.0,
268        'passing_test_2': 2.0,
269        'passing_test_3': 2.0,
270        'passing_test_4': 2.0,
271        'passing_test_5': 1.0,
272        'passing_test_6': 1.0,
273        'passing_test_7': 1.0,
274        'passing_test_8': 1.0,
275        'passing_test_9': 0.5,
276      }
277    }
278    temp_file = tempfile.NamedTemporaryFile(delete=False)
279    temp_file.close()
280    temp_file_name = temp_file.name
281    with open(temp_file_name, 'w') as f:
282      json.dump(mock_test_results, f)
283    return temp_file_name
284
285  @decorators.Disabled('chromeos')  # crbug.com/696553
286  def testSplittingShardsByTimes(self):
287    temp_file_name = self.writeMockTestResultsFile()
288    # It seems that the sorting order of the first four tests above is:
289    #   passing_test_0, Test1, Test2, Test3
290    # This is probably because the relative order of the "fixed" tests
291    # (starting with "Test") and the generated ones ("passing_") is
292    # not well defined, and the sorting is stable afterward.  The
293    # expectations have been adjusted for this fact.
294    try:
295      self.baseShardingTest(
296        4, 0, [],
297        ['browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_0',
298         'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_1',
299         'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_5',
300         'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_9'
301        ], temp_file_name)
302      self.baseShardingTest(
303        4, 1, [],
304        ['browser_tests.simple_sharding_test.SimpleShardingTest.Test1',
305         'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_2',
306         'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_6'
307        ], temp_file_name)
308      self.baseShardingTest(
309        4, 2, [],
310        ['browser_tests.simple_sharding_test.SimpleShardingTest.Test2',
311         'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_3',
312         'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_7'
313        ], temp_file_name)
314      self.baseShardingTest(
315        4, 3, [],
316        ['browser_tests.simple_sharding_test.SimpleShardingTest.Test3',
317         'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_4',
318         'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_8'
319        ], temp_file_name)
320    finally:
321      os.remove(temp_file_name)
322
323  @decorators.Disabled('chromeos')  # crbug.com/696553
324  def testFilteringAfterSharding(self):
325    temp_file_name = self.writeMockTestResultsFile()
326    try:
327      self.baseShardingTest(
328        4, 1, [],
329        ['browser_tests.simple_sharding_test.SimpleShardingTest.Test1',
330         'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_2',
331         'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_6'
332        ], temp_file_name,
333        opt_test_filter='(Test1|passing_test_2|passing_test_6)',
334        opt_filter_tests_after_sharding=True)
335    finally:
336      os.remove(temp_file_name)
337
338  def testMedianComputation(self):
339    self.assertEquals(2.0, run_browser_tests._MedianTestTime(
340      {'test1': 2.0, 'test2': 7.0, 'test3': 1.0}))
341    self.assertEquals(2.0, run_browser_tests._MedianTestTime(
342      {'test1': 2.0}))
343    self.assertEquals(0.0, run_browser_tests._MedianTestTime({}))
344    self.assertEqual(4.0, run_browser_tests._MedianTestTime(
345      {'test1': 2.0, 'test2': 6.0, 'test3': 1.0, 'test4': 8.0}))
346
347
348class Algebra(
349    serially_executed_browser_test_case.SeriallyExecutedBrowserTestCase):
350
351  @classmethod
352  def GenerateTestCases_Simple(cls, options):
353    del options  # Unused.
354    yield 'testOne', (1, 2)
355    yield 'testTwo', (3, 3)
356
357  def Simple(self, x, y):
358    self.assertEquals(x, y)
359
360  def TestNumber(self):
361    self.assertEquals(0, 1)
362
363
364class ErrorneousGeometric(
365    serially_executed_browser_test_case.SeriallyExecutedBrowserTestCase):
366
367  @classmethod
368  def GenerateTestCases_Compare(cls, options):
369    del options  # Unused.
370    assert False, 'I am a problematic generator'
371    yield 'testBasic', ('square', 'circle')
372
373  def Compare(self, x, y):
374    self.assertEquals(x, y)
375
376  def TestAngle(self):
377    self.assertEquals(90, 450)
378
379class TestLoadAllTestModules(unittest.TestCase):
380  def testLoadAllTestsInModule(self):
381    context = browser_test_context.TypTestContext()
382    context.finder_options = options_for_unittests.GetCopy()
383    context.test_class = Algebra
384    context.test_case_ids_to_run.add(
385      'telemetry.testing.browser_test_runner_unittest.Algebra.TestNumber')
386    context.test_case_ids_to_run.add(
387      'telemetry.testing.browser_test_runner_unittest.Algebra.testOne')
388    context.Freeze()
389    browser_test_context._global_test_context = context
390    try:
391      # This should not invoke GenerateTestCases of ErrorneousGeometric class,
392      # otherwise that would throw Exception.
393      tests = serially_executed_browser_test_case.LoadAllTestsInModule(
394          sys.modules[__name__])
395      self.assertEquals(sorted([t.id() for t in tests]),
396          ['telemetry.testing.browser_test_runner_unittest.Algebra.TestNumber',
397           'telemetry.testing.browser_test_runner_unittest.Algebra.testOne'])
398    finally:
399      browser_test_context._global_test_context = None
400