17757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# Copyright (C) 2012 Google Inc. All rights reserved. 27757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# 37757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# Redistribution and use in source and binary forms, with or without 47757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# modification, are permitted provided that the following conditions are 57757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# met: 67757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# 77757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# * Redistributions of source code must retain the above copyright 87757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# notice, this list of conditions and the following disclaimer. 97757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# * Redistributions in binary form must reproduce the above 107757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# copyright notice, this list of conditions and the following disclaimer 117757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# in the documentation and/or other materials provided with the 127757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# distribution. 137757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# * Neither the name of Google Inc. nor the names of its 147757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# contributors may be used to endorse or promote products derived from 157757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# this software without specific prior written permission. 167757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# 177757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 187757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 197757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 207757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 217757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 227757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 237757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 247757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 257757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 267757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 277757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 287757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 297757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch"""Integration tests for run_perf_tests.""" 307757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 317757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochimport StringIO 327757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochimport datetime 337757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochimport json 347757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochimport re 357757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochimport unittest2 as unittest 367757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 377757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochfrom webkitpy.common.host_mock import MockHost 387757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochfrom webkitpy.common.system.outputcapture import OutputCapture 397757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochfrom webkitpy.layout_tests.port.driver import DriverOutput 407757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochfrom webkitpy.layout_tests.port.test import TestPort 417757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochfrom webkitpy.performance_tests.perftest import ChromiumStylePerfTest 427757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochfrom webkitpy.performance_tests.perftest import PerfTest 437757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochfrom webkitpy.performance_tests.perftestsrunner import PerfTestsRunner 447757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 457757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 467757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochclass InspectorPassTestData: 477757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch text = 'RESULT group_name: test_name= 42 ms' 487757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output = """Running inspector/pass.html (2 of 2) 497757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen MurdochRESULT group_name: test_name= 42 ms 507757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen MurdochFinished: 0.1 s 517757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 527757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch""" 537757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 547757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 557757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochclass EventTargetWrapperTestData: 567757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch text = """Running 20 times 577757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen MurdochIgnoring warm-up run (1502) 587757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch1504 597757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch1505 607757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch1510 617757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch1504 627757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch1507 637757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch1509 647757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch1510 657757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch1487 667757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch1488 677757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch1472 687757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch1472 697757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch1488 707757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch1473 717757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch1472 727757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch1475 737757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch1487 747757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch1486 757757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch1486 767757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch1475 777757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch1471 787757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 797757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen MurdochTime: 807757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochvalues 1486, 1471, 1510, 1505, 1478, 1490 ms 817757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochavg 1490 ms 827757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochmedian 1488 ms 837757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochstdev 15.13935 ms 847757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochmin 1471 ms 857757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochmax 1510 ms 867757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch""" 877757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 887757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output = """Running Bindings/event-target-wrapper.html (1 of 2) 897757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen MurdochRESULT Bindings: event-target-wrapper: Time= 1490.0 ms 907757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochmedian= 1488.0 ms, stdev= 14.11751 ms, min= 1471.0 ms, max= 1510.0 ms 917757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen MurdochFinished: 0.1 s 927757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 937757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch""" 947757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 957757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch results = {'url': 'http://trac.webkit.org/browser/trunk/PerformanceTests/Bindings/event-target-wrapper.html', 967757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 'metrics': {'Time': {'current': [[1486.0, 1471.0, 1510.0, 1505.0, 1478.0, 1490.0]] * 4}}} 977757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 987757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 997757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochclass SomeParserTestData: 1007757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch text = """Running 20 times 1017757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen MurdochIgnoring warm-up run (1115) 1027757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1037757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen MurdochTime: 1047757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochvalues 1080, 1120, 1095, 1101, 1104 ms 1057757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochavg 1100 ms 1067757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochmedian 1101 ms 1077757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochstdev 14.50861 ms 1087757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochmin 1080 ms 1097757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochmax 1120 ms 1107757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch""" 1117757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1127757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output = """Running Parser/some-parser.html (2 of 2) 1137757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen MurdochRESULT Parser: some-parser: Time= 1100.0 ms 1147757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochmedian= 1101.0 ms, stdev= 13.31402 ms, min= 1080.0 ms, max= 1120.0 ms 1157757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen MurdochFinished: 0.1 s 1167757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1177757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch""" 1187757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1197757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1207757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochclass MemoryTestData: 1217757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch text = """Running 20 times 1227757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen MurdochIgnoring warm-up run (1115) 1237757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1247757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen MurdochTime: 1257757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochvalues 1080, 1120, 1095, 1101, 1104 ms 1267757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochavg 1100 ms 1277757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochmedian 1101 ms 1287757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochstdev 14.50861 ms 1297757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochmin 1080 ms 1307757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochmax 1120 ms 1317757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1327757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen MurdochJS Heap: 1337757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochvalues 825000, 811000, 848000, 837000, 829000 bytes 1347757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochavg 830000 bytes 1357757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochmedian 829000 bytes 1367757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochstdev 13784.04875 bytes 1377757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochmin 811000 bytes 1387757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochmax 848000 bytes 1397757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1407757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen MurdochMalloc: 1417757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochvalues 529000, 511000, 548000, 536000, 521000 bytes 1427757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochavg 529000 bytes 1437757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochmedian 529000 bytes 1447757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochstdev 14124.44689 bytes 1457757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochmin 511000 bytes 1467757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochmax 548000 bytes 1477757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch""" 1487757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1497757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output = """Running 1 tests 1507757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen MurdochRunning Parser/memory-test.html (1 of 1) 1517757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen MurdochRESULT Parser: memory-test: Time= 1100.0 ms 1527757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochmedian= 1101.0 ms, stdev= 13.31402 ms, min= 1080.0 ms, max= 1120.0 ms 1537757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen MurdochRESULT Parser: memory-test: JSHeap= 830000.0 bytes 1547757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochmedian= 829000.0 bytes, stdev= 12649.11064 bytes, min= 811000.0 bytes, max= 848000.0 bytes 1557757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen MurdochRESULT Parser: memory-test: Malloc= 529000.0 bytes 1567757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochmedian= 529000.0 bytes, stdev= 12961.48139 bytes, min= 511000.0 bytes, max= 548000.0 bytes 1577757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen MurdochFinished: 0.1 s 1587757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch""" 1597757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1607757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch results = {'current': [[1080, 1120, 1095, 1101, 1104]] * 4} 1617757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch js_heap_results = {'current': [[825000, 811000, 848000, 837000, 829000]] * 4} 1627757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch malloc_results = {'current': [[529000, 511000, 548000, 536000, 521000]] * 4} 1637757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1647757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1657757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochclass TestDriver: 1667757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def run_test(self, driver_input, stop_when_done): 1677757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch text = '' 1687757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch timeout = False 1697757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch crash = False 1707757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if driver_input.test_name.endswith('pass.html'): 1717757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch text = InspectorPassTestData.text 1727757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch elif driver_input.test_name.endswith('timeout.html'): 1737757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch timeout = True 1747757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch elif driver_input.test_name.endswith('failed.html'): 1757757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch text = None 1767757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch elif driver_input.test_name.endswith('tonguey.html'): 1777757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch text = 'we are not expecting an output from perf tests but RESULT blablabla' 1787757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch elif driver_input.test_name.endswith('crash.html'): 1797757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch crash = True 1807757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch elif driver_input.test_name.endswith('event-target-wrapper.html'): 1817757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch text = EventTargetWrapperTestData.text 1827757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch elif driver_input.test_name.endswith('some-parser.html'): 1837757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch text = SomeParserTestData.text 1847757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch elif driver_input.test_name.endswith('memory-test.html'): 1857757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch text = MemoryTestData.text 1867757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return DriverOutput(text, '', '', '', crash=crash, timeout=timeout) 1877757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1887757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def start(self): 1897757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch """do nothing""" 1907757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1917757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def stop(self): 1927757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch """do nothing""" 1937757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1947757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1957757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochclass MainTest(unittest.TestCase): 1967757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def _normalize_output(self, log): 1977757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return re.sub(r'(stdev=\s+\d+\.\d{5})\d+', r'\1', re.sub(r'Finished: [0-9\.]+ s', 'Finished: 0.1 s', log)) 1987757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1997757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def _load_output_json(self, runner): 2007757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch json_content = runner._host.filesystem.read_text_file(runner._output_json_path()) 2017757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return json.loads(re.sub(r'("stdev":\s*\d+\.\d{5})\d+', r'\1', json_content)) 2027757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2037757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def create_runner(self, args=[], driver_class=TestDriver): 2047757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch options, parsed_args = PerfTestsRunner._parse_args(args) 2057757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch test_port = TestPort(host=MockHost(), options=options) 2067757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch test_port.create_driver = lambda worker_number=None, no_timeout=False: driver_class() 2077757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2087757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner = PerfTestsRunner(args=args, port=test_port) 2097757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner._host.filesystem.maybe_make_directory(runner._base_path, 'inspector') 2107757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner._host.filesystem.maybe_make_directory(runner._base_path, 'Bindings') 2117757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner._host.filesystem.maybe_make_directory(runner._base_path, 'Parser') 2127757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2137757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return runner, test_port 2147757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2157757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def run_test(self, test_name): 2167757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner() 2177757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch tests = [ChromiumStylePerfTest(port, test_name, runner._host.filesystem.join('some-dir', test_name))] 2187757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return runner._run_tests_set(tests) == 0 2197757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2207757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_passing_test(self): 2217757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertTrue(self.run_test('pass.html')) 2227757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2237757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_silent_test(self): 2247757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertFalse(self.run_test('silent.html')) 2257757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2267757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_failed_test(self): 2277757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertFalse(self.run_test('failed.html')) 2287757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2297757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_tonguey_test(self): 2307757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertFalse(self.run_test('tonguey.html')) 2317757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2327757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_timeout_test(self): 2337757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertFalse(self.run_test('timeout.html')) 2347757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2357757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_crash_test(self): 2367757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertFalse(self.run_test('crash.html')) 2377757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2387757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def _tests_for_runner(self, runner, test_names): 2397757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch filesystem = runner._host.filesystem 2407757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch tests = [] 2417757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch for test in test_names: 2427757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch path = filesystem.join(runner._base_path, test) 2437757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch dirname = filesystem.dirname(path) 2447757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if test.startswith('inspector/'): 2457757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch tests.append(ChromiumStylePerfTest(runner._port, test, path)) 2467757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch else: 2477757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch tests.append(PerfTest(runner._port, test, path)) 2487757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return tests 2497757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2507757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_test_set(self): 2517757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner() 2527757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch tests = self._tests_for_runner(runner, ['inspector/pass.html', 'inspector/silent.html', 'inspector/failed.html', 2537757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 'inspector/tonguey.html', 'inspector/timeout.html', 'inspector/crash.html']) 2547757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output = OutputCapture() 2557757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output.capture_output() 2567757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch try: 2577757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch unexpected_result_count = runner._run_tests_set(tests) 2587757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch finally: 2597757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch stdout, stderr, log = output.restore_output() 2607757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(unexpected_result_count, len(tests) - 1) 2617757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertTrue('\nRESULT group_name: test_name= 42 ms\n' in log) 2627757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2637757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_test_set_kills_drt_per_run(self): 2647757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2657757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch class TestDriverWithStopCount(TestDriver): 2667757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch stop_count = 0 2677757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def stop(self): 2687757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch TestDriverWithStopCount.stop_count += 1 2697757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2707757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner(driver_class=TestDriverWithStopCount) 2717757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2727757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch tests = self._tests_for_runner(runner, ['inspector/pass.html', 'inspector/silent.html', 'inspector/failed.html', 2737757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 'inspector/tonguey.html', 'inspector/timeout.html', 'inspector/crash.html']) 2747757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch unexpected_result_count = runner._run_tests_set(tests) 2757757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2767757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(TestDriverWithStopCount.stop_count, 6) 2777757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2787757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_test_set_for_parser_tests(self): 2797757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner() 2807757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch tests = self._tests_for_runner(runner, ['Bindings/event-target-wrapper.html', 'Parser/some-parser.html']) 2817757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output = OutputCapture() 2827757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output.capture_output() 2837757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch try: 2847757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch unexpected_result_count = runner._run_tests_set(tests) 2857757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch finally: 2867757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch stdout, stderr, log = output.restore_output() 2877757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(unexpected_result_count, 0) 2887757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(self._normalize_output(log), EventTargetWrapperTestData.output + SomeParserTestData.output) 2897757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2907757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_memory_test(self): 2917757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner_and_setup_results_template() 2927757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner._timestamp = 123456789 2937757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch port.host.filesystem.write_text_file(runner._base_path + '/Parser/memory-test.html', 'some content') 2947757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2957757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output = OutputCapture() 2967757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output.capture_output() 2977757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch try: 2987757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch unexpected_result_count = runner.run() 2997757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch finally: 3007757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch stdout, stderr, log = output.restore_output() 3017757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(unexpected_result_count, 0) 3027757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(self._normalize_output(log), MemoryTestData.output + '\nMOCK: user.open_url: file://...\n') 3037757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch parser_tests = self._load_output_json(runner)[0]['tests']['Parser']['tests'] 3047757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(parser_tests['memory-test']['metrics']['Time'], MemoryTestData.results) 3057757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(parser_tests['memory-test']['metrics']['JSHeap'], MemoryTestData.js_heap_results) 3067757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(parser_tests['memory-test']['metrics']['Malloc'], MemoryTestData.malloc_results) 3077757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3087757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def _test_run_with_json_output(self, runner, filesystem, upload_succeeds=False, results_shown=True, expected_exit_code=0, repeat=1, compare_logs=True): 3097757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch filesystem.write_text_file(runner._base_path + '/inspector/pass.html', 'some content') 3107757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch filesystem.write_text_file(runner._base_path + '/Bindings/event-target-wrapper.html', 'some content') 3117757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3127757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch uploaded = [False] 3137757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3147757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def mock_upload_json(hostname, json_path, host_path=None): 3157757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch # FIXME: Get rid of the hard-coded perf.webkit.org once we've completed the transition. 3167757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertIn(hostname, ['some.host']) 3177757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertIn(json_path, ['/mock-checkout/output.json']) 3187757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertIn(host_path, [None, '/api/report']) 3197757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch uploaded[0] = upload_succeeds 3207757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return upload_succeeds 3217757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3227757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner._upload_json = mock_upload_json 3237757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner._timestamp = 123456789 3247757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner._utc_timestamp = datetime.datetime(2013, 2, 8, 15, 19, 37, 460000) 3257757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output_capture = OutputCapture() 3267757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output_capture.capture_output() 3277757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch try: 3287757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(runner.run(), expected_exit_code) 3297757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch finally: 3307757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch stdout, stderr, logs = output_capture.restore_output() 3317757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3327757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if not expected_exit_code and compare_logs: 3337757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch expected_logs = '' 3347757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch for i in xrange(repeat): 3357757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runs = ' (Run %d of %d)' % (i + 1, repeat) if repeat > 1 else '' 3367757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch expected_logs += 'Running 2 tests%s\n' % runs + EventTargetWrapperTestData.output + InspectorPassTestData.output 3377757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if results_shown: 3387757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch expected_logs += 'MOCK: user.open_url: file://...\n' 3397757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(self._normalize_output(logs), expected_logs) 3407757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3417757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(uploaded[0], upload_succeeds) 3427757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3437757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return logs 3447757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3457757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch _event_target_wrapper_and_inspector_results = { 3467757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "Bindings": 3477757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch {"url": "http://trac.webkit.org/browser/trunk/PerformanceTests/Bindings", 3487757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "tests": {"event-target-wrapper": EventTargetWrapperTestData.results}}} 3497757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3507757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_with_json_output(self): 3517757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json', 3527757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch '--test-results-server=some.host']) 3537757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._test_run_with_json_output(runner, port.host.filesystem, upload_succeeds=True) 3547757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(self._load_output_json(runner), [{ 3557757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "buildTime": "2013-02-08T15:19:37.460000", "tests": self._event_target_wrapper_and_inspector_results, 3567757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "revisions": {"blink": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}}}]) 3577757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3587757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch filesystem = port.host.filesystem 3597757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertTrue(filesystem.isfile(runner._output_json_path())) 3607757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertTrue(filesystem.isfile(filesystem.splitext(runner._output_json_path())[0] + '.html')) 3617757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3627757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_with_description(self): 3637757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json', 3647757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch '--test-results-server=some.host', '--description', 'some description']) 3657757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._test_run_with_json_output(runner, port.host.filesystem, upload_succeeds=True) 3667757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(self._load_output_json(runner), [{ 3677757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "buildTime": "2013-02-08T15:19:37.460000", "description": "some description", 3687757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "tests": self._event_target_wrapper_and_inspector_results, 3697757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "revisions": {"blink": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}}}]) 3707757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3717757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def create_runner_and_setup_results_template(self, args=[]): 3727757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner(args) 3737757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch filesystem = port.host.filesystem 3747757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch filesystem.write_text_file(runner._base_path + '/resources/results-template.html', 3757757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 'BEGIN<script src="%AbsolutePathToWebKitTrunk%/some.js"></script>' 3767757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch '<script src="%AbsolutePathToWebKitTrunk%/other.js"></script><script>%PeformanceTestsResultsJSON%</script>END') 3777757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch filesystem.write_text_file(runner._base_path + '/Dromaeo/resources/dromaeo/web/lib/jquery-1.6.4.js', 'jquery content') 3787757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return runner, port 3797757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3807757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_respects_no_results(self): 3817757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner(args=['--output-json-path=/mock-checkout/output.json', 3827757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch '--test-results-server=some.host', '--no-results']) 3837757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._test_run_with_json_output(runner, port.host.filesystem, upload_succeeds=False, results_shown=False) 3847757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertFalse(port.host.filesystem.isfile('/mock-checkout/output.json')) 3857757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3867757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_generates_json_by_default(self): 3877757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner_and_setup_results_template() 3887757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch filesystem = port.host.filesystem 3897757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output_json_path = runner._output_json_path() 3907757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch results_page_path = filesystem.splitext(output_json_path)[0] + '.html' 3917757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3927757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertFalse(filesystem.isfile(output_json_path)) 3937757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertFalse(filesystem.isfile(results_page_path)) 3947757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3957757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._test_run_with_json_output(runner, port.host.filesystem) 3967757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3977757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(self._load_output_json(runner), [{ 3987757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "buildTime": "2013-02-08T15:19:37.460000", "tests": self._event_target_wrapper_and_inspector_results, 3997757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "revisions": {"blink": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}}}]) 4007757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 4017757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertTrue(filesystem.isfile(output_json_path)) 4027757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertTrue(filesystem.isfile(results_page_path)) 4037757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 4047757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_merges_output_by_default(self): 4057757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner_and_setup_results_template() 4067757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch filesystem = port.host.filesystem 4077757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output_json_path = runner._output_json_path() 4087757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 4097757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch filesystem.write_text_file(output_json_path, '[{"previous": "results"}]') 4107757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 4117757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._test_run_with_json_output(runner, port.host.filesystem) 4127757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 4137757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(self._load_output_json(runner), [{"previous": "results"}, { 4147757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "buildTime": "2013-02-08T15:19:37.460000", "tests": self._event_target_wrapper_and_inspector_results, 4157757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "revisions": {"blink": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}}}]) 4167757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertTrue(filesystem.isfile(filesystem.splitext(output_json_path)[0] + '.html')) 4177757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 4187757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_respects_reset_results(self): 4197757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner_and_setup_results_template(args=["--reset-results"]) 4207757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch filesystem = port.host.filesystem 4217757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output_json_path = runner._output_json_path() 4227757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 4237757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch filesystem.write_text_file(output_json_path, '[{"previous": "results"}]') 4247757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 4257757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._test_run_with_json_output(runner, port.host.filesystem) 4267757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 4277757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(self._load_output_json(runner), [{ 4287757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "buildTime": "2013-02-08T15:19:37.460000", "tests": self._event_target_wrapper_and_inspector_results, 4297757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "revisions": {"blink": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}}}]) 4307757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertTrue(filesystem.isfile(filesystem.splitext(output_json_path)[0] + '.html')) 4317757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch pass 4327757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 4337757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_generates_and_show_results_page(self): 4347757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json']) 4357757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch page_shown = [] 4367757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch port.show_results_html_file = lambda path: page_shown.append(path) 4377757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch filesystem = port.host.filesystem 4387757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._test_run_with_json_output(runner, filesystem, results_shown=False) 4397757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 4407757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch expected_entry = {"buildTime": "2013-02-08T15:19:37.460000", "tests": self._event_target_wrapper_and_inspector_results, 4417757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "revisions": {"blink": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}}} 4427757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 4437757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.maxDiff = None 4447757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(runner._output_json_path(), '/mock-checkout/output.json') 4457757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(self._load_output_json(runner), [expected_entry]) 4467757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(filesystem.read_text_file('/mock-checkout/output.html'), 4477757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 'BEGIN<script src="/test.checkout/some.js"></script><script src="/test.checkout/other.js"></script>' 4487757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch '<script>%s</script>END' % port.host.filesystem.read_text_file(runner._output_json_path())) 4497757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(page_shown[0], '/mock-checkout/output.html') 4507757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 4517757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._test_run_with_json_output(runner, filesystem, results_shown=False) 4527757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(runner._output_json_path(), '/mock-checkout/output.json') 4537757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(self._load_output_json(runner), [expected_entry, expected_entry]) 4547757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(filesystem.read_text_file('/mock-checkout/output.html'), 4557757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 'BEGIN<script src="/test.checkout/some.js"></script><script src="/test.checkout/other.js"></script>' 4567757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch '<script>%s</script>END' % port.host.filesystem.read_text_file(runner._output_json_path())) 4577757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 4587757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_respects_no_show_results(self): 4597757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch show_results_html_file = lambda path: page_shown.append(path) 4607757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 4617757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json']) 4627757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch page_shown = [] 4637757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch port.show_results_html_file = show_results_html_file 4647757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._test_run_with_json_output(runner, port.host.filesystem, results_shown=False) 4657757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(page_shown[0], '/mock-checkout/output.html') 4667757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 4677757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json', 4687757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch '--no-show-results']) 4697757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch page_shown = [] 4707757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch port.show_results_html_file = show_results_html_file 4717757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._test_run_with_json_output(runner, port.host.filesystem, results_shown=False) 4727757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(page_shown, []) 4737757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 4747757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_with_bad_output_json(self): 4757757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json']) 4767757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch port.host.filesystem.write_text_file('/mock-checkout/output.json', 'bad json') 4777757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._test_run_with_json_output(runner, port.host.filesystem, expected_exit_code=PerfTestsRunner.EXIT_CODE_BAD_MERGE) 4787757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch port.host.filesystem.write_text_file('/mock-checkout/output.json', '{"another bad json": "1"}') 4797757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._test_run_with_json_output(runner, port.host.filesystem, expected_exit_code=PerfTestsRunner.EXIT_CODE_BAD_MERGE) 4807757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 4817757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_with_slave_config_json(self): 4827757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json', 4837757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch '--slave-config-json-path=/mock-checkout/slave-config.json', '--test-results-server=some.host']) 4847757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch port.host.filesystem.write_text_file('/mock-checkout/slave-config.json', '{"key": "value"}') 4857757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._test_run_with_json_output(runner, port.host.filesystem, upload_succeeds=True) 4867757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(self._load_output_json(runner), [{ 4877757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "buildTime": "2013-02-08T15:19:37.460000", "tests": self._event_target_wrapper_and_inspector_results, 4887757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "revisions": {"blink": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}}, "builderKey": "value"}]) 4897757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 4907757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_with_bad_slave_config_json(self): 4917757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json', 4927757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch '--slave-config-json-path=/mock-checkout/slave-config.json', '--test-results-server=some.host']) 4937757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch logs = self._test_run_with_json_output(runner, port.host.filesystem, expected_exit_code=PerfTestsRunner.EXIT_CODE_BAD_SOURCE_JSON) 4947757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertTrue('Missing slave configuration JSON file: /mock-checkout/slave-config.json' in logs) 4957757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch port.host.filesystem.write_text_file('/mock-checkout/slave-config.json', 'bad json') 4967757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._test_run_with_json_output(runner, port.host.filesystem, expected_exit_code=PerfTestsRunner.EXIT_CODE_BAD_SOURCE_JSON) 4977757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch port.host.filesystem.write_text_file('/mock-checkout/slave-config.json', '["another bad json"]') 4987757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._test_run_with_json_output(runner, port.host.filesystem, expected_exit_code=PerfTestsRunner.EXIT_CODE_BAD_SOURCE_JSON) 4997757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 5007757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_with_multiple_repositories(self): 5017757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json', 5027757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch '--test-results-server=some.host']) 5037757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch port.repository_paths = lambda: [('webkit', '/mock-checkout'), ('some', '/mock-checkout/some')] 5047757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._test_run_with_json_output(runner, port.host.filesystem, upload_succeeds=True) 5057757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(self._load_output_json(runner), [{ 5067757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "buildTime": "2013-02-08T15:19:37.460000", "tests": self._event_target_wrapper_and_inspector_results, 5077757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "revisions": {"webkit": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}, 5087757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "some": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}}}]) 5097757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 5107757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_with_upload_json(self): 5117757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json', 5127757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch '--test-results-server', 'some.host', '--platform', 'platform1', '--builder-name', 'builder1', '--build-number', '123']) 5137757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 5147757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._test_run_with_json_output(runner, port.host.filesystem, upload_succeeds=True) 5157757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch generated_json = json.loads(port.host.filesystem.files['/mock-checkout/output.json']) 5167757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(generated_json[0]['platform'], 'platform1') 5177757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(generated_json[0]['builderName'], 'builder1') 5187757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(generated_json[0]['buildNumber'], 123) 5197757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 5207757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._test_run_with_json_output(runner, port.host.filesystem, upload_succeeds=False, expected_exit_code=PerfTestsRunner.EXIT_CODE_FAILED_UPLOADING) 5217757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 5227757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_with_upload_json_should_generate_perf_webkit_json(self): 5237757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json', 5247757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch '--test-results-server', 'some.host', '--platform', 'platform1', '--builder-name', 'builder1', '--build-number', '123', 5257757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch '--slave-config-json-path=/mock-checkout/slave-config.json']) 5267757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch port.host.filesystem.write_text_file('/mock-checkout/slave-config.json', '{"key": "value1"}') 5277757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 5287757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._test_run_with_json_output(runner, port.host.filesystem, upload_succeeds=True) 5297757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch generated_json = json.loads(port.host.filesystem.files['/mock-checkout/output.json']) 5307757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertTrue(isinstance(generated_json, list)) 5317757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(len(generated_json), 1) 5327757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 5337757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output = generated_json[0] 5347757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.maxDiff = None 5357757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(output['platform'], 'platform1') 5367757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(output['buildNumber'], 123) 5377757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(output['buildTime'], '2013-02-08T15:19:37.460000') 5387757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(output['builderName'], 'builder1') 5397757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(output['builderKey'], 'value1') 5407757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(output['revisions'], {'blink': {'revision': '5678', 'timestamp': '2013-02-01 08:48:05 +0000'}}) 5417757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(output['tests'].keys(), ['Bindings']) 5427757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(sorted(output['tests']['Bindings'].keys()), ['tests', 'url']) 5437757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(output['tests']['Bindings']['url'], 'http://trac.webkit.org/browser/trunk/PerformanceTests/Bindings') 5447757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(output['tests']['Bindings']['tests'].keys(), ['event-target-wrapper']) 5457757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(output['tests']['Bindings']['tests']['event-target-wrapper'], { 5467757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 'url': 'http://trac.webkit.org/browser/trunk/PerformanceTests/Bindings/event-target-wrapper.html', 5477757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 'metrics': {'Time': {'current': [[1486.0, 1471.0, 1510.0, 1505.0, 1478.0, 1490.0]] * 4}}}) 5487757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 5497757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_with_repeat(self): 5507757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.maxDiff = None 5517757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json', 5527757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch '--test-results-server=some.host', '--repeat', '5']) 5537757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._test_run_with_json_output(runner, port.host.filesystem, upload_succeeds=True, repeat=5) 5547757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(self._load_output_json(runner), [ 5557757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch {"buildTime": "2013-02-08T15:19:37.460000", 5567757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "tests": self._event_target_wrapper_and_inspector_results, 5577757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "revisions": {"blink": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}}}, 5587757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch {"buildTime": "2013-02-08T15:19:37.460000", 5597757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "tests": self._event_target_wrapper_and_inspector_results, 5607757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "revisions": {"blink": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}}}, 5617757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch {"buildTime": "2013-02-08T15:19:37.460000", 5627757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "tests": self._event_target_wrapper_and_inspector_results, 5637757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "revisions": {"blink": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}}}, 5647757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch {"buildTime": "2013-02-08T15:19:37.460000", 5657757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "tests": self._event_target_wrapper_and_inspector_results, 5667757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "revisions": {"blink": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}}}, 5677757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch {"buildTime": "2013-02-08T15:19:37.460000", 5687757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "tests": self._event_target_wrapper_and_inspector_results, 5697757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch "revisions": {"blink": {"timestamp": "2013-02-01 08:48:05 +0000", "revision": "5678"}}}]) 5707757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 5717757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def test_run_with_test_runner_count(self): 5727757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runner, port = self.create_runner_and_setup_results_template(args=['--output-json-path=/mock-checkout/output.json', 5737757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch '--test-runner-count=3']) 5747757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._test_run_with_json_output(runner, port.host.filesystem, compare_logs=False) 5757757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch generated_json = json.loads(port.host.filesystem.files['/mock-checkout/output.json']) 5767757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertTrue(isinstance(generated_json, list)) 5777757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(len(generated_json), 1) 5787757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 5797757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output = generated_json[0]['tests']['Bindings']['tests']['event-target-wrapper']['metrics']['Time']['current'] 5807757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(len(output), 3) 5817757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch expectedMetrics = EventTargetWrapperTestData.results['metrics']['Time']['current'][0] 5827757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch for metrics in output: 5837757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self.assertEqual(metrics, expectedMetrics) 584