1# Copyright (C) 2010 Google Inc. All rights reserved.
2#
3# Redistribution and use in source and binary forms, with or without
4# modification, are permitted provided that the following conditions are
5# met:
6#
7#     * Redistributions of source code must retain the above copyright
8# notice, this list of conditions and the following disclaimer.
9#     * Redistributions in binary form must reproduce the above
10# copyright notice, this list of conditions and the following disclaimer
11# in the documentation and/or other materials provided with the
12# distribution.
13#     * Neither the name of Google Inc. nor the names of its
14# contributors may be used to endorse or promote products derived from
15# this software without specific prior written permission.
16#
17# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
18# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
19# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
20# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
21# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
22# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
23# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
24# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
25# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
27# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
28
29try:
30    import jsonresults
31    from jsonresults import JsonResults
32except ImportError:
33    print "ERROR: Add the TestResultServer, google_appengine and yaml/lib directories to your PYTHONPATH"
34
35import unittest
36
37
38JSON_RESULTS_TEMPLATE = (
39    '{"Webkit":{'
40    '"allFixableCount":[[TESTDATA_COUNT]],'
41    '"buildNumbers":[[TESTDATA_BUILDNUMBERS]],'
42    '"chromeRevision":[[TESTDATA_CHROMEREVISION]],'
43    '"deferredCounts":[[TESTDATA_COUNTS]],'
44    '"fixableCount":[[TESTDATA_COUNT]],'
45    '"fixableCounts":[[TESTDATA_COUNTS]],'
46    '"secondsSinceEpoch":[[TESTDATA_TIMES]],'
47    '"tests":{[TESTDATA_TESTS]},'
48    '"webkitRevision":[[TESTDATA_WEBKITREVISION]],'
49    '"wontfixCounts":[[TESTDATA_COUNTS]]'
50    '},'
51    '"version":3'
52    '}')
53
54JSON_RESULTS_COUNTS_TEMPLATE = (
55    '{'
56    '"C":[TESTDATA],'
57    '"F":[TESTDATA],'
58    '"I":[TESTDATA],'
59    '"O":[TESTDATA],'
60    '"P":[TESTDATA],'
61    '"T":[TESTDATA],'
62    '"X":[TESTDATA],'
63    '"Z":[TESTDATA]}')
64
65JSON_RESULTS_TESTS_TEMPLATE = (
66    '"[TESTDATA_TEST_NAME]":{'
67    '"results":[[TESTDATA_TEST_RESULTS]],'
68    '"times":[[TESTDATA_TEST_TIMES]]}')
69
70JSON_RESULTS_PREFIX = "ADD_RESULTS("
71JSON_RESULTS_SUFFIX = ");"
72
73JSON_RESULTS_TEST_LIST_TEMPLATE = (
74    '{"Webkit":{"tests":{[TESTDATA_TESTS]}}}')
75
76
77class JsonResultsTest(unittest.TestCase):
78    def setUp(self):
79        self._builder = "Webkit"
80
81    def _make_test_json(self, test_data):
82        if not test_data:
83            return JSON_RESULTS_PREFIX + JSON_RESULTS_SUFFIX
84
85        (builds, tests) = test_data
86        if not builds or not tests:
87            return JSON_RESULTS_PREFIX + JSON_RESULTS_SUFFIX
88
89        json = JSON_RESULTS_TEMPLATE
90
91        counts = []
92        build_numbers = []
93        webkit_revision = []
94        chrome_revision = []
95        times = []
96        for build in builds:
97            counts.append(JSON_RESULTS_COUNTS_TEMPLATE.replace("[TESTDATA]", build))
98            build_numbers.append("1000%s" % build)
99            webkit_revision.append("2000%s" % build)
100            chrome_revision.append("3000%s" % build)
101            times.append("100000%s000" % build)
102
103        json = json.replace("[TESTDATA_COUNTS]", ",".join(counts))
104        json = json.replace("[TESTDATA_COUNT]", ",".join(builds))
105        json = json.replace("[TESTDATA_BUILDNUMBERS]", ",".join(build_numbers))
106        json = json.replace("[TESTDATA_WEBKITREVISION]", ",".join(webkit_revision))
107        json = json.replace("[TESTDATA_CHROMEREVISION]", ",".join(chrome_revision))
108        json = json.replace("[TESTDATA_TIMES]", ",".join(times))
109
110        json_tests = []
111        for test in tests:
112            t = JSON_RESULTS_TESTS_TEMPLATE.replace("[TESTDATA_TEST_NAME]", test[0])
113            t = t.replace("[TESTDATA_TEST_RESULTS]", test[1])
114            t = t.replace("[TESTDATA_TEST_TIMES]", test[2])
115            json_tests.append(t)
116
117        json = json.replace("[TESTDATA_TESTS]", ",".join(json_tests))
118
119        return JSON_RESULTS_PREFIX + json + JSON_RESULTS_SUFFIX
120
121    def _test_merge(self, aggregated_data, incremental_data, expected_data, max_builds=jsonresults.JSON_RESULTS_MAX_BUILDS):
122        aggregated_results = self._make_test_json(aggregated_data)
123        incremental_results = self._make_test_json(incremental_data)
124        merged_results = JsonResults.merge(self._builder,
125            aggregated_results, incremental_results, max_builds,
126            sort_keys=True)
127
128        if expected_data:
129            expected_results = self._make_test_json(expected_data)
130            self.assertEquals(merged_results, expected_results)
131        else:
132            self.assertFalse(merged_results)
133
134    def _test_get_test_list(self, input_data, expected_data):
135        input_results = self._make_test_json(input_data)
136
137        json_tests = []
138        for test in expected_data:
139            json_tests.append("\"" + test + "\":{}")
140
141        expected_results = JSON_RESULTS_PREFIX + \
142            JSON_RESULTS_TEST_LIST_TEMPLATE.replace(
143                "[TESTDATA_TESTS]", ",".join(json_tests)) + \
144            JSON_RESULTS_SUFFIX
145
146        actual_results = JsonResults.get_test_list(self._builder, input_results)
147        self.assertEquals(actual_results, expected_results)
148
149    def test(self):
150        # Empty incremental results json.
151        # Nothing to merge.
152        self._test_merge(
153            # Aggregated results
154            (["2", "1"], [["001.html", "[200,\"F\"]", "[200,0]"]]),
155            # Incremental results
156            None,
157            # Expect no merge happens.
158            None)
159
160        # No actual incremental test results (only prefix and suffix) to merge.
161        # Nothing to merge.
162        self._test_merge(
163            # Aggregated results
164            (["2", "1"], [["001.html", "[200,\"F\"]", "[200,0]"]]),
165            # Incremental results
166            ([], []),
167            # Expected no merge happens.
168            None)
169
170        # No existing aggregated results.
171        # Merged results == new incremental results.
172        self._test_merge(
173            # Aggregated results
174            None,
175            # Incremental results
176            (["2", "1"], [["001.html", "[200,\"F\"]", "[200,0]"]]),
177            # Expected results
178            (["2", "1"], [["001.html", "[200,\"F\"]", "[200,0]"]]))
179
180        # Single test for single run.
181        # Incremental results has the latest build and same test results for
182        # that run.
183        # Insert the incremental results at the first place and sum number
184        # of runs for "P" (200 + 1) to get merged results.
185        self._test_merge(
186            # Aggregated results
187            (["2", "1"], [["001.html", "[200,\"F\"]", "[200,0]"]]),
188            # Incremental results
189            (["3"], [["001.html", "[1,\"F\"]", "[1,0]"]]),
190            # Expected results
191            (["3", "2", "1"], [["001.html", "[201,\"F\"]", "[201,0]"]]))
192
193        # Single test for single run.
194        # Incremental results has the latest build but different test results
195        # for that run.
196        # Insert the incremental results at the first place.
197        self._test_merge(
198            # Aggregated results
199            (["2", "1"], [["001.html", "[200,\"F\"]", "[200,0]"]]),
200            # Incremental results
201            (["3"], [["001.html", "[1, \"I\"]", "[1,1]"]]),
202            # Expected results
203            (["3", "2", "1"], [["001.html", "[1,\"I\"],[200,\"F\"]", "[1,1],[200,0]"]]))
204
205        # Single test for single run.
206        # Incremental results has the latest build but different test results
207        # for that run.
208        self._test_merge(
209            # Aggregated results
210            (["2", "1"], [["001.html", "[200,\"F\"],[10,\"I\"]", "[200,0],[10,1]"]]),
211            # Incremental results
212            (["3"], [["001.html", "[1,\"I\"]", "[1,1]"]]),
213            # Expected results
214            (["3", "2", "1"], [["001.html", "[1,\"I\"],[200,\"F\"],[10,\"I\"]", "[1,1],[200,0],[10,1]"]]))
215
216        # Multiple tests for single run.
217        # All tests have incremental updates.
218        self._test_merge(
219            # Aggregated results
220            (["2", "1"], [["001.html", "[200,\"F\"]", "[200,0]"], ["002.html", "[100,\"I\"]", "[100,1]"]]),
221            # Incremental results
222            (["3"], [["001.html", "[1,\"F\"]", "[1,0]"], ["002.html", "[1,\"I\"]", "[1,1]"]]),
223            # Expected results
224            (["3", "2", "1"], [["001.html", "[201,\"F\"]", "[201,0]"], ["002.html", "[101,\"I\"]", "[101,1]"]]))
225
226        # Multiple tests for single run.
227        self._test_merge(
228            # Aggregated results
229            (["2", "1"], [["001.html", "[200,\"F\"]", "[200,0]"], ["002.html", "[100,\"I\"]", "[100,1]"]]),
230            # Incremental results
231            (["3"], [["002.html", "[1,\"I\"]", "[1,1]"]]),
232            # Expected results
233            (["3", "2", "1"], [["001.html", "[1,\"N\"],[200,\"F\"]", "[201,0]"], ["002.html", "[101,\"I\"]", "[101,1]"]]))
234
235        # Single test for multiple runs.
236        self._test_merge(
237            # Aggregated results
238            (["2", "1"], [["001.html", "[200,\"F\"]", "[200,0]"]]),
239            # Incremental results
240            (["4", "3"], [["001.html", "[2, \"I\"]", "[2,2]"]]),
241            # Expected results
242            (["4", "3", "2", "1"], [["001.html", "[2,\"I\"],[200,\"F\"]", "[2,2],[200,0]"]]))
243
244        # Multiple tests for multiple runs.
245        self._test_merge(
246            # Aggregated results
247            (["2", "1"], [["001.html", "[200,\"F\"]", "[200,0]"], ["002.html", "[10,\"Z\"]", "[10,0]"]]),
248            # Incremental results
249            (["4", "3"], [["001.html", "[2, \"I\"]", "[2,2]"], ["002.html", "[1,\"C\"]", "[1,1]"]]),
250            # Expected results
251            (["4", "3", "2", "1"], [["001.html", "[2,\"I\"],[200,\"F\"]", "[2,2],[200,0]"], ["002.html", "[1,\"C\"],[10,\"Z\"]", "[1,1],[10,0]"]]))
252
253        # Test the build in incremental results is older than the most recent
254        # build in aggregated results.
255        # The incremental results should be dropped and no merge happens.
256        self._test_merge(
257            # Aggregated results
258            (["3", "1"], [["001.html", "[200,\"F\"]", "[200,0]"]]),
259            # Incremental results
260            (["2"], [["001.html", "[1, \"F\"]", "[1,0]"]]),
261            # Expected no merge happens.
262            None)
263
264        # Test the build in incremental results is same as the build in
265        # aggregated results.
266        # The incremental results should be dropped and no merge happens.
267        self._test_merge(
268            # Aggregated results
269            (["2", "1"], [["001.html", "[200,\"F\"]", "[200,0]"]]),
270            # Incremental results
271            (["3", "2"], [["001.html", "[2, \"F\"]", "[2,0]"]]),
272            # Expected no merge happens.
273            None)
274
275        # Remove test where there is no data in all runs.
276        self._test_merge(
277            # Aggregated results
278            (["2", "1"], [["001.html", "[200,\"N\"]", "[200,0]"], ["002.html", "[10,\"F\"]", "[10,0]"]]),
279            # Incremental results
280            (["3"], [["001.html", "[1,\"N\"]", "[1,0]"], ["002.html", "[1,\"P\"]", "[1,0]"]]),
281            # Expected results
282            (["3", "2", "1"], [["002.html", "[1,\"P\"],[10,\"F\"]", "[11,0]"]]))
283
284        # Remove test where all run pass and max running time < 1 seconds
285        self._test_merge(
286            # Aggregated results
287            (["2", "1"], [["001.html", "[200,\"P\"]", "[200,0]"], ["002.html", "[10,\"F\"]", "[10,0]"]]),
288            # Incremental results
289            (["3"], [["001.html", "[1,\"P\"]", "[1,0]"], ["002.html", "[1,\"P\"]", "[1,0]"]]),
290            # Expected results
291            (["3", "2", "1"], [["002.html", "[1,\"P\"],[10,\"F\"]", "[11,0]"]]))
292
293        # Do not remove test where all run pass but max running time >= 1 seconds
294        self._test_merge(
295            # Aggregated results
296            (["2", "1"], [["001.html", "[200,\"P\"]", "[200,0]"], ["002.html", "[10,\"F\"]", "[10,0]"]]),
297            # Incremental results
298            (["3"], [["001.html", "[1,\"P\"]", "[1,1]"], ["002.html", "[1,\"P\"]", "[1,0]"]]),
299            # Expected results
300            (["3", "2", "1"], [["001.html", "[201,\"P\"]", "[1,1],[200,0]"], ["002.html", "[1,\"P\"],[10,\"F\"]", "[11,0]"]]))
301
302        # Remove items from test results and times that exceed the max number
303        # of builds to track.
304        max_builds = str(jsonresults.JSON_RESULTS_MAX_BUILDS)
305        self._test_merge(
306            # Aggregated results
307            (["2", "1"], [["001.html", "[" + max_builds + ",\"F\"],[1,\"I\"]", "[" + max_builds + ",0],[1,1]"]]),
308            # Incremental results
309            (["3"], [["001.html", "[1,\"T\"]", "[1,1]"]]),
310            # Expected results
311            (["3", "2", "1"], [["001.html", "[1,\"T\"],[" + max_builds + ",\"F\"]", "[1,1],[" + max_builds + ",0]"]]))
312
313        # Remove items from test results and times that exceed the max number
314        # of builds to track, using smaller threshold.
315        max_builds = str(jsonresults.JSON_RESULTS_MAX_BUILDS_SMALL)
316        self._test_merge(
317            # Aggregated results
318            (["2", "1"], [["001.html", "[" + max_builds + ",\"F\"],[1,\"I\"]", "[" + max_builds + ",0],[1,1]"]]),
319            # Incremental results
320            (["3"], [["001.html", "[1,\"T\"]", "[1,1]"]]),
321            # Expected results
322            (["3", "2", "1"], [["001.html", "[1,\"T\"],[" + max_builds + ",\"F\"]", "[1,1],[" + max_builds + ",0]"]]),
323            int(max_builds))
324
325        # Test that merging in a new result of the same type as the last result
326        # causes old results to fall off.
327        max_builds = str(jsonresults.JSON_RESULTS_MAX_BUILDS_SMALL)
328        self._test_merge(
329            # Aggregated results
330            (["2", "1"], [["001.html", "[" + max_builds + ",\"F\"],[1,\"N\"]", "[" + max_builds + ",0],[1,1]"]]),
331            # Incremental results
332            (["3"], [["001.html", "[1,\"F\"]", "[1,0]"]]),
333            # Expected results
334            (["3", "2", "1"], [["001.html", "[" + max_builds + ",\"F\"]", "[" + max_builds + ",0]"]]),
335            int(max_builds))
336
337        # Get test name list only. Don't include non-test-list data and
338        # of test result details.
339        self._test_get_test_list(
340            # Input results
341            (["3", "2", "1"], [["001.html", "[200,\"P\"]", "[200,0]"], ["002.html", "[10,\"F\"]", "[10,0]"]]),
342            # Expected results
343            ["001.html", "002.html"])
344
345if __name__ == '__main__':
346    unittest.main()
347