1#!/usr/bin/env python
2# Copyright 2014 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6
7"""Unit tests for instrumentation.TestRunner."""
8
9# pylint: disable=W0212
10
11import os
12import sys
13import unittest
14
15from pylib import constants
16from pylib.base import base_test_result
17from pylib.instrumentation import test_runner
18
19sys.path.append(os.path.join(
20    constants.DIR_SOURCE_ROOT, 'third_party', 'pymock'))
21import mock  # pylint: disable=F0401
22
23
24class InstrumentationTestRunnerTest(unittest.TestCase):
25
26  def setUp(self):
27    options = mock.Mock()
28    options.tool = ''
29    package = mock.Mock()
30    self.instance = test_runner.TestRunner(options, None, 0, package)
31
32  def testParseAmInstrumentRawOutput_nothing(self):
33    code, result, statuses = (
34        test_runner.TestRunner._ParseAmInstrumentRawOutput(['']))
35    self.assertEqual(None, code)
36    self.assertEqual([], result)
37    self.assertEqual([], statuses)
38
39  def testParseAmInstrumentRawOutput_noMatchingStarts(self):
40    raw_output = [
41      '',
42      'this.is.a.test.package.TestClass:.',
43      'Test result for =.',
44      'Time: 1.234',
45      '',
46      'OK (1 test)',
47    ]
48
49    code, result, statuses = (
50        test_runner.TestRunner._ParseAmInstrumentRawOutput(raw_output))
51    self.assertEqual(None, code)
52    self.assertEqual([], result)
53    self.assertEqual([], statuses)
54
55  def testParseAmInstrumentRawOutput_resultAndCode(self):
56    raw_output = [
57      'INSTRUMENTATION_RESULT: foo',
58      'bar',
59      'INSTRUMENTATION_CODE: -1',
60    ]
61
62    code, result, _ = (
63        test_runner.TestRunner._ParseAmInstrumentRawOutput(raw_output))
64    self.assertEqual(-1, code)
65    self.assertEqual(['foo', 'bar'], result)
66
67  def testParseAmInstrumentRawOutput_oneStatus(self):
68    raw_output = [
69      'INSTRUMENTATION_STATUS: foo=1',
70      'INSTRUMENTATION_STATUS: bar=hello',
71      'INSTRUMENTATION_STATUS: world=false',
72      'INSTRUMENTATION_STATUS: class=this.is.a.test.package.TestClass',
73      'INSTRUMENTATION_STATUS: test=testMethod',
74      'INSTRUMENTATION_STATUS_CODE: 0',
75    ]
76
77    _, _, statuses = (
78        test_runner.TestRunner._ParseAmInstrumentRawOutput(raw_output))
79
80    expected = [
81      (0, {
82        'foo': ['1'],
83        'bar': ['hello'],
84        'world': ['false'],
85        'class': ['this.is.a.test.package.TestClass'],
86        'test': ['testMethod'],
87      })
88    ]
89    self.assertEqual(expected, statuses)
90
91  def testParseAmInstrumentRawOutput_multiStatus(self):
92    raw_output = [
93      'INSTRUMENTATION_STATUS: class=foo',
94      'INSTRUMENTATION_STATUS: test=bar',
95      'INSTRUMENTATION_STATUS_CODE: 1',
96      'INSTRUMENTATION_STATUS: test_skipped=true',
97      'INSTRUMENTATION_STATUS_CODE: 0',
98      'INSTRUMENTATION_STATUS: class=hello',
99      'INSTRUMENTATION_STATUS: test=world',
100      'INSTRUMENTATION_STATUS: stack=',
101      'foo/bar.py (27)',
102      'hello/world.py (42)',
103      'test/file.py (1)',
104      'INSTRUMENTATION_STATUS_CODE: -1',
105    ]
106
107    _, _, statuses = (
108        test_runner.TestRunner._ParseAmInstrumentRawOutput(raw_output))
109
110    expected = [
111      (1, {'class': ['foo'], 'test': ['bar'],}),
112      (0, {'test_skipped': ['true']}),
113      (-1, {
114        'class': ['hello'],
115        'test': ['world'],
116        'stack': ['', 'foo/bar.py (27)', 'hello/world.py (42)',
117                  'test/file.py (1)'],
118      }),
119    ]
120    self.assertEqual(expected, statuses)
121
122  def testParseAmInstrumentRawOutput_statusResultAndCode(self):
123    raw_output = [
124      'INSTRUMENTATION_STATUS: class=foo',
125      'INSTRUMENTATION_STATUS: test=bar',
126      'INSTRUMENTATION_STATUS_CODE: 1',
127      'INSTRUMENTATION_RESULT: hello',
128      'world',
129      '',
130      '',
131      'INSTRUMENTATION_CODE: 0',
132    ]
133
134    code, result, statuses = (
135        test_runner.TestRunner._ParseAmInstrumentRawOutput(raw_output))
136
137    self.assertEqual(0, code)
138    self.assertEqual(['hello', 'world', '', ''], result)
139    self.assertEqual([(1, {'class': ['foo'], 'test': ['bar']})], statuses)
140
141  def testGenerateTestResult_noStatus(self):
142    result = self.instance._GenerateTestResult(
143        'test.package.TestClass#testMethod', [], 0, 1000)
144    self.assertEqual('test.package.TestClass#testMethod', result.GetName())
145    self.assertEqual(base_test_result.ResultType.UNKNOWN, result.GetType())
146    self.assertEqual('', result.GetLog())
147    self.assertEqual(1000, result.GetDur())
148
149  def testGenerateTestResult_testPassed(self):
150    statuses = [
151      (1, {
152        'class': ['test.package.TestClass'],
153        'test': ['testMethod'],
154      }),
155      (0, {
156        'class': ['test.package.TestClass'],
157        'test': ['testMethod'],
158      }),
159    ]
160    result = self.instance._GenerateTestResult(
161        'test.package.TestClass#testMethod', statuses, 0, 1000)
162    self.assertEqual(base_test_result.ResultType.PASS, result.GetType())
163
164  def testGenerateTestResult_testSkipped_first(self):
165    statuses = [
166      (0, {
167        'test_skipped': ['true'],
168      }),
169      (1, {
170        'class': ['test.package.TestClass'],
171        'test': ['testMethod'],
172      }),
173      (0, {
174        'class': ['test.package.TestClass'],
175        'test': ['testMethod'],
176      }),
177    ]
178    result = self.instance._GenerateTestResult(
179        'test.package.TestClass#testMethod', statuses, 0, 1000)
180    self.assertEqual(base_test_result.ResultType.SKIP, result.GetType())
181
182  def testGenerateTestResult_testSkipped_last(self):
183    statuses = [
184      (1, {
185        'class': ['test.package.TestClass'],
186        'test': ['testMethod'],
187      }),
188      (0, {
189        'class': ['test.package.TestClass'],
190        'test': ['testMethod'],
191      }),
192      (0, {
193        'test_skipped': ['true'],
194      }),
195    ]
196    result = self.instance._GenerateTestResult(
197        'test.package.TestClass#testMethod', statuses, 0, 1000)
198    self.assertEqual(base_test_result.ResultType.SKIP, result.GetType())
199
200  def testGenerateTestResult_testSkipped_false(self):
201    statuses = [
202      (0, {
203        'test_skipped': ['false'],
204      }),
205      (1, {
206        'class': ['test.package.TestClass'],
207        'test': ['testMethod'],
208      }),
209      (0, {
210        'class': ['test.package.TestClass'],
211        'test': ['testMethod'],
212      }),
213    ]
214    result = self.instance._GenerateTestResult(
215        'test.package.TestClass#testMethod', statuses, 0, 1000)
216    self.assertEqual(base_test_result.ResultType.PASS, result.GetType())
217
218  def testGenerateTestResult_testFailed(self):
219    statuses = [
220      (1, {
221        'class': ['test.package.TestClass'],
222        'test': ['testMethod'],
223      }),
224      (-2, {
225        'class': ['test.package.TestClass'],
226        'test': ['testMethod'],
227      }),
228    ]
229    result = self.instance._GenerateTestResult(
230        'test.package.TestClass#testMethod', statuses, 0, 1000)
231    self.assertEqual(base_test_result.ResultType.FAIL, result.GetType())
232
233  def testGenerateTestResult_testCrashed(self):
234    self.instance.test_pkg.GetPackageName = mock.Mock(
235        return_value='generate.test.result.test.package')
236    self.instance.device.old_interface.DismissCrashDialogIfNeeded = mock.Mock(
237        return_value='generate.test.result.test.package')
238    statuses = [
239      (1, {
240        'class': ['test.package.TestClass'],
241        'test': ['testMethod'],
242      }),
243      (-1, {
244        'class': ['test.package.TestClass'],
245        'test': ['testMethod'],
246        'stack': ['', 'foo/bar.py (27)', 'hello/world.py (42)'],
247      }),
248    ]
249    result = self.instance._GenerateTestResult(
250        'test.package.TestClass#testMethod', statuses, 0, 1000)
251    self.assertEqual(base_test_result.ResultType.CRASH, result.GetType())
252    self.assertEqual('\nfoo/bar.py (27)\nhello/world.py (42)', result.GetLog())
253
254  def test_RunTest_verifyAdbShellCommand(self):
255    self.instance.options.test_runner = 'MyTestRunner'
256    self.instance.device.RunShellCommand = mock.Mock()
257    self.instance.test_pkg.GetPackageName = mock.Mock(
258        return_value='test.package')
259    self.instance._GetInstrumentationArgs = mock.Mock(
260        return_value={'test_arg_key': 'test_arg_value'})
261    self.instance._RunTest('test.package.TestClass#testMethod', 100)
262    self.instance.device.RunShellCommand.assert_called_with(
263        ['am', 'instrument', '-r',
264         '-e', 'test_arg_key', "'test_arg_value'",
265         '-e', 'class', "'test.package.TestClass#testMethod'",
266         '-w', 'test.package/MyTestRunner'],
267        timeout=100, retries=0)
268
269if __name__ == '__main__':
270  unittest.main(verbosity=2)
271
272