15c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Copyright (C) 2010 Google Inc. All rights reserved. 25c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 35c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Redistribution and use in source and binary forms, with or without 45c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# modification, are permitted provided that the following conditions are 55c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# met: 65c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 75c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Redistributions of source code must retain the above copyright 85c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# notice, this list of conditions and the following disclaimer. 95c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Redistributions in binary form must reproduce the above 105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# copyright notice, this list of conditions and the following disclaimer 115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# in the documentation and/or other materials provided with the 125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# distribution. 135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Neither the name of Google Inc. nor the names of its 145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# contributors may be used to endorse or promote products derived from 155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# this software without specific prior written permission. 165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 29e38fbeeb576b5094e34e038ab88d9d6a5c5c2214Torne (Richard Coles)import unittest 305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.common.checkout.baselineoptimizer import BaselineOptimizer 32f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles)from webkitpy.common.checkout.scm.scm_mock import MockSCM 33f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles)from webkitpy.common.host_mock import MockHost 345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.common.net.buildbot.buildbot_mock import MockBuilder 35fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdochfrom webkitpy.common.net.layouttestresults import LayoutTestResults 36f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles)from webkitpy.common.system.executive_mock import MockExecutive 375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.common.system.executive_mock import MockExecutive2 38f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles)from webkitpy.common.system.outputcapture import OutputCapture 395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.tool.commands.rebaseline import * 405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.tool.mocktool import MockTool, MockOptions 415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class _BaseTestCase(unittest.TestCase): 445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) MOCK_WEB_RESULT = 'MOCK Web result, convert 404 to None=True' 455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) WEB_PREFIX = 'http://example.com/f/builders/WebKit Mac10.7/results/layout-test-results' 465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) command_constructor = None 485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def setUp(self): 505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.tool = MockTool() 51926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.command = self.command_constructor() # lint warns that command_constructor might not be set, but this is intentional; pylint: disable=E1102 525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.command.bind_to_tool(self.tool) 535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.lion_port = self.tool.port_factory.get_from_builder_name("WebKit Mac10.7") 5453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self.lion_expectations_path = self.lion_port.path_to_generic_test_expectations_file() 557242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci self.tool.filesystem.write_text_file(self.tool.filesystem.join(self.lion_port.layout_tests_dir(), "VirtualTestSuites"), 567242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci '[]') 575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 58c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # FIXME: crbug.com/279494. We should override builders._exact_matches 59c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # here to point to a set of test ports and restore the value in 60c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # tearDown(), and that way the individual tests wouldn't have to worry 61c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # about it. 625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _expand(self, path): 645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if self.tool.filesystem.isabs(path): 655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return path 665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return self.tool.filesystem.join(self.lion_port.layout_tests_dir(), path) 675c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _read(self, path): 695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return self.tool.filesystem.read_text_file(self._expand(path)) 705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 715c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _write(self, path, contents): 725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.tool.filesystem.write_text_file(self._expand(path), contents) 735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _zero_out_test_expectations(self): 755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for port_name in self.tool.port_factory.all_port_names(): 765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) port = self.tool.port_factory.get(port_name) 775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) for path in port.expectations_files(): 785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._write(path, '') 795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.tool.filesystem.written_files = {} 805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 813c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch def _setup_mock_builder_data(self): 823c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch data = LayoutTestResults.results_from_string("""ADD_RESULTS({ 833c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch "tests": { 843c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch "userscripts": { 853c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch "first-test.html": { 863c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch "expected": "PASS", 873c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch "actual": "IMAGE+TEXT" 883c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch }, 893c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch "second-test.html": { 903c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch "expected": "FAIL", 913c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch "actual": "IMAGE+TEXT" 923c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch } 933c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch } 943c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch } 953c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch});""") 96c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # FIXME: crbug.com/279494 - we shouldn't be mixing mock and real builder names. 973c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch for builder in ['MOCK builder', 'MOCK builder (Debug)', 'WebKit Mac10.7']: 983c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self.command._builder_data[builder] = data 993c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch 1005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 101e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdochclass TestCopyExistingBaselinesInternal(_BaseTestCase): 102e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) command_constructor = CopyExistingBaselinesInternal 1035c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1045c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def setUp(self): 105e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch super(TestCopyExistingBaselinesInternal, self).setUp() 1065c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 107e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def test_copying_overwritten_baseline(self): 1085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.tool.executive = MockExecutive2() 1095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 110c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # FIXME: crbug.com/279494. it's confusing that this is the test- port, and not the regular lion port. Really all of the tests should be using the test ports. 1115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) port = self.tool.port_factory.get('test-mac-snowleopard') 1125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-mac-snowleopard/failures/expected/image-expected.txt'), 'original snowleopard result') 1135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) old_exact_matches = builders._exact_matches 1155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) oc = OutputCapture() 1165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) try: 1175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) builders._exact_matches = { 1185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 1195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 1205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 1215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 122e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch options = MockOptions(builder="MOCK SnowLeopard", suffixes="txt", verbose=True, test="failures/expected/image.html", results_directory=None) 1235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) oc.capture_output() 1255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.command.execute(options, [], self.tool) 1265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) finally: 1275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) out, _, _ = oc.restore_output() 1285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) builders._exact_matches = old_exact_matches 1295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 130926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-mac-leopard/failures/expected/image-expected.txt')), 'original snowleopard result') 131f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [], "delete": []}\n') 13253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 133e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def test_copying_overwritten_baseline_to_multiple_locations(self): 13453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self.tool.executive = MockExecutive2() 13553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 136c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # FIXME: crbug.com/279494. it's confusing that this is the test- port, and not the regular win port. Really all of the tests should be using the test ports. 13753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) port = self.tool.port_factory.get('test-win-win7') 13853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt'), 'original win7 result') 13953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 14053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) old_exact_matches = builders._exact_matches 14153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) oc = OutputCapture() 14253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) try: 14353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) builders._exact_matches = { 14453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 14553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) "MOCK Linux": {"port_name": "test-linux-x86_64", "specifiers": set(["mock-specifier"])}, 14653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) "MOCK Win7": {"port_name": "test-win-win7", "specifiers": set(["mock-specifier"])}, 14753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) } 14853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 149e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch options = MockOptions(builder="MOCK Win7", suffixes="txt", verbose=True, test="failures/expected/image.html", results_directory=None) 15053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 15153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) oc.capture_output() 15253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self.command.execute(options, [], self.tool) 15353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) finally: 15453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) out, _, _ = oc.restore_output() 15553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) builders._exact_matches = old_exact_matches 15653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 15753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-linux-x86_64/failures/expected/image-expected.txt')), 'original win7 result') 158e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/mac-leopard/userscripts/another-test-expected.txt'))) 159f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [], "delete": []}\n') 16053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 161e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def test_no_copy_existing_baseline(self): 16253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self.tool.executive = MockExecutive2() 16353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 16453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # FIXME: it's confusing that this is the test- port, and not the regular win port. Really all of the tests should be using the test ports. 16553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) port = self.tool.port_factory.get('test-win-win7') 16653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt'), 'original win7 result') 16753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 16853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) old_exact_matches = builders._exact_matches 16953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) oc = OutputCapture() 17053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) try: 17153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) builders._exact_matches = { 17253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 17353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) "MOCK Linux": {"port_name": "test-linux-x86_64", "specifiers": set(["mock-specifier"])}, 17453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) "MOCK Win7": {"port_name": "test-win-win7", "specifiers": set(["mock-specifier"])}, 17553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) } 17653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 177e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch options = MockOptions(builder="MOCK Win7", suffixes="txt", verbose=True, test="failures/expected/image.html", results_directory=None) 17853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 17953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) oc.capture_output() 18053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self.command.execute(options, [], self.tool) 18153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) finally: 18253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) out, _, _ = oc.restore_output() 18353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) builders._exact_matches = old_exact_matches 18453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 18553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-linux-x86_64/failures/expected/image-expected.txt')), 'original win7 result') 186e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt')), 'original win7 result') 187e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/mac-leopard/userscripts/another-test-expected.txt'))) 188f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [], "delete": []}\n') 189e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 190e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) def test_no_copy_skipped_test(self): 191e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) self.tool.executive = MockExecutive2() 192e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) 193e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) port = self.tool.port_factory.get('test-win-win7') 194e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) fs = self.tool.filesystem 195e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) self._write(fs.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt'), 'original win7 result') 196e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) expectations_path = fs.join(port.path_to_generic_test_expectations_file()) 197e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) self._write(expectations_path, ( 198e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) "[ Win ] failures/expected/image.html [ Failure ]\n" 199e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) "[ Linux ] failures/expected/image.html [ Skip ]\n")) 200e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) old_exact_matches = builders._exact_matches 201e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) oc = OutputCapture() 202e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) try: 203e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) builders._exact_matches = { 204e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) "MOCK Linux": {"port_name": "test-linux-x86_64", "specifiers": set(["mock-specifier"])}, 205e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 206e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) "MOCK Win7": {"port_name": "test-win-win7", "specifiers": set(["mock-specifier"])}, 207e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) } 208e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) 209e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) options = MockOptions(builder="MOCK Win7", suffixes="txt", verbose=True, test="failures/expected/image.html", results_directory=None) 210e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) 211e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) oc.capture_output() 212e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) self.command.execute(options, [], self.tool) 213e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) finally: 214e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) out, _, _ = oc.restore_output() 215e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) builders._exact_matches = old_exact_matches 216e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) 217e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) self.assertFalse(fs.exists(fs.join(port.layout_tests_dir(), 'platform/test-linux-x86_64/failures/expected/image-expected.txt'))) 218e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) self.assertEqual(self._read(fs.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt')), 219e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) 'original win7 result') 220e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) 221e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 222e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdochclass TestRebaselineTest(_BaseTestCase): 223e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch command_constructor = RebaselineTest # AKA webkit-patch rebaseline-test-internal 224e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 225e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def setUp(self): 226e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch super(TestRebaselineTest, self).setUp() 227e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.options = MockOptions(builder="WebKit Mac10.7", test="userscripts/another-test.html", suffixes="txt", results_directory=None) 228e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 229e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def test_baseline_directory(self): 230e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch command = self.command 2311e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) self.assertMultiLineEqual(command._baseline_directory("WebKit Mac10.7"), "/mock-checkout/third_party/WebKit/LayoutTests/platform/mac-lion") 2321e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) self.assertMultiLineEqual(command._baseline_directory("WebKit Mac10.6"), "/mock-checkout/third_party/WebKit/LayoutTests/platform/mac-snowleopard") 233e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 234e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def test_rebaseline_updates_expectations_file_noop(self): 235e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self._zero_out_test_expectations() 236e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self._write(self.lion_expectations_path, """Bug(B) [ Mac Linux XP Debug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ] 237e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben MurdochBug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ] 238e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch""") 239e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self._write("fast/dom/Window/window-postmessage-clone-really-deep-array.html", "Dummy test contents") 240e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self._write("fast/css/large-list-of-rules-crash.html", "Dummy test contents") 241e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self._write("userscripts/another-test.html", "Dummy test contents") 242e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 243e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.options.suffixes = "png,wav,txt" 244e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.command._rebaseline_test_and_update_expectations(self.options) 245e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 246e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.assertItemsEqual(self.tool.web.urls_fetched, 247e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch [self.WEB_PREFIX + '/userscripts/another-test-actual.png', 248e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.WEB_PREFIX + '/userscripts/another-test-actual.wav', 249e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.WEB_PREFIX + '/userscripts/another-test-actual.txt']) 250e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch new_expectations = self._read(self.lion_expectations_path) 251e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.assertMultiLineEqual(new_expectations, """Bug(B) [ Mac Linux XP Debug ] fast/dom/Window/window-postmessage-clone-really-deep-array.html [ Pass ] 252e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben MurdochBug(A) [ Debug ] : fast/css/large-list-of-rules-crash.html [ Failure ] 253e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch""") 254e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 255e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def test_rebaseline_test(self): 256e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.command._rebaseline_test("WebKit Linux", "userscripts/another-test.html", "txt", self.WEB_PREFIX) 257e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.assertItemsEqual(self.tool.web.urls_fetched, [self.WEB_PREFIX + '/userscripts/another-test-actual.txt']) 258e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 259e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def test_rebaseline_test_with_results_directory(self): 260e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self._write("userscripts/another-test.html", "test data") 261e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self._write(self.lion_expectations_path, "Bug(x) [ Mac ] userscripts/another-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/another-test.html [ ImageOnlyFailure ]\n") 262e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.options.results_directory = '/tmp' 263e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.command._rebaseline_test_and_update_expectations(self.options) 264e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.assertItemsEqual(self.tool.web.urls_fetched, ['file:///tmp/userscripts/another-test-actual.txt']) 265e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 266e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def test_rebaseline_reftest(self): 267e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self._write("userscripts/another-test.html", "test data") 268e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self._write("userscripts/another-test-expected.html", "generic result") 269e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch OutputCapture().assert_outputs(self, self.command._rebaseline_test_and_update_expectations, args=[self.options], 270e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch expected_logs="Cannot rebaseline reftest: userscripts/another-test.html\n") 271f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.assertDictEqual(self.command._scm_changes, {'add': [], 'remove-lines': [], "delete": []}) 272e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 273e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def test_rebaseline_test_and_print_scm_changes(self): 274e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.command._print_scm_changes = True 275e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.command._scm_changes = {'add': [], 'delete': []} 276e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.tool._scm.exists = lambda x: False 277e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 278e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.command._rebaseline_test("WebKit Linux", "userscripts/another-test.html", "txt", None) 279e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 2801e202183a5dc46166763171984b285173f8585e5Torne (Richard Coles) self.assertDictEqual(self.command._scm_changes, {'add': ['/mock-checkout/third_party/WebKit/LayoutTests/platform/linux/userscripts/another-test-expected.txt'], 'delete': []}) 28153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 28253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def test_rebaseline_test_internal_with_port_that_lacks_buildbot(self): 28353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self.tool.executive = MockExecutive2() 28453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 28553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) # FIXME: it's confusing that this is the test- port, and not the regular win port. Really all of the tests should be using the test ports. 286521d96ec04ace82590870fb04353ec4f82bb150fTorne (Richard Coles) port = self.tool.port_factory.get('test-win-win7') 287521d96ec04ace82590870fb04353ec4f82bb150fTorne (Richard Coles) self._write(port._filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt'), 'original win7 result') 28853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 28953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) old_exact_matches = builders._exact_matches 29053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) oc = OutputCapture() 29153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) try: 29253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) builders._exact_matches = { 29353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) "MOCK XP": {"port_name": "test-win-xp"}, 294521d96ec04ace82590870fb04353ec4f82bb150fTorne (Richard Coles) "MOCK Win7": {"port_name": "test-win-win7"}, 29553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) } 29653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 297521d96ec04ace82590870fb04353ec4f82bb150fTorne (Richard Coles) options = MockOptions(optimize=True, builder="MOCK Win7", suffixes="txt", 298e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch verbose=True, test="failures/expected/image.html", results_directory=None) 29953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 30053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) oc.capture_output() 30153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self.command.execute(options, [], self.tool) 30253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) finally: 30353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) out, _, _ = oc.restore_output() 30453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) builders._exact_matches = old_exact_matches 30553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 306521d96ec04ace82590870fb04353ec4f82bb150fTorne (Richard Coles) self.assertMultiLineEqual(self._read(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-win-win7/failures/expected/image-expected.txt')), 'MOCK Web result, convert 404 to None=True') 30753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(port.layout_tests_dir(), 'platform/test-win-xp/failures/expected/image-expected.txt'))) 308f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.assertMultiLineEqual(out, '{"add": [], "remove-lines": [{"test": "failures/expected/image.html", "builder": "MOCK Win7"}], "delete": []}\n') 3095c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 311e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdochclass TestAbstractParallelRebaselineCommand(_BaseTestCase): 312e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch command_constructor = AbstractParallelRebaselineCommand 313e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 314e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def test_builders_to_fetch_from(self): 315e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch old_exact_matches = builders._exact_matches 316e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch try: 317e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch builders._exact_matches = { 318e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "MOCK XP": {"port_name": "test-win-xp"}, 319e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "MOCK Win7": {"port_name": "test-win-win7"}, 320e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "MOCK Win7 (dbg)(1)": {"port_name": "test-win-win7"}, 321e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "MOCK Win7 (dbg)(2)": {"port_name": "test-win-win7"}, 322e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch } 323e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 324e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch builders_to_fetch = self.command._builders_to_fetch_from(["MOCK XP", "MOCK Win7 (dbg)(1)", "MOCK Win7 (dbg)(2)", "MOCK Win7"]) 325e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.assertEqual(builders_to_fetch, ["MOCK XP", "MOCK Win7"]) 326e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch finally: 327e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch builders._exact_matches = old_exact_matches 328e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 329e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 3305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class TestRebaselineJson(_BaseTestCase): 3315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) command_constructor = RebaselineJson 3325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def setUp(self): 3345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) super(TestRebaselineJson, self).setUp() 3355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.tool.executive = MockExecutive2() 3365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.old_exact_matches = builders._exact_matches 3375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) builders._exact_matches = { 338e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "MOCK builder": {"port_name": "test-mac-snowleopard"}, 339e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "MOCK builder (Debug)": {"port_name": "test-mac-snowleopard"}, 3405c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 3415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def tearDown(self): 3435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) builders._exact_matches = self.old_exact_matches 3445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) super(TestRebaselineJson, self).tearDown() 3455c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3468abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) def test_rebaseline_test_passes_on_all_builders(self): 3478abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) self._setup_mock_builder_data() 3488abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) 3498abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) def builder_data(): 3508abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) self.command._builder_data['MOCK builder'] = LayoutTestResults.results_from_string("""ADD_RESULTS({ 3518abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) "tests": { 3528abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) "userscripts": { 3538abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) "first-test.html": { 3548abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) "expected": "NEEDSREBASELINE", 3558abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) "actual": "PASS" 3568abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) } 3578abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) } 3588abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) } 3598abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles)});""") 3608abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) return self.command._builder_data 3618abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) 3628abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) self.command.builder_data = builder_data 3638abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) 3648abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) options = MockOptions(optimize=True, verbose=True, results_directory=None) 365f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 366f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self._write(self.lion_expectations_path, "Bug(x) userscripts/first-test.html [ ImageOnlyFailure ]\n") 3678abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) self._write("userscripts/first-test.html", "Dummy test contents") 368f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 3698abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) self.command._rebaseline(options, {"userscripts/first-test.html": {"MOCK builder": ["txt", "png"]}}) 3708abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) 3718abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) # Note that we have one run_in_parallel() call followed by a run_command() 3728abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) self.assertEqual(self.tool.executive.calls, 373c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) [[['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', '', 'userscripts/first-test.html', '--verbose']]]) 3748abfc5808a4e34d6e03867af8bc440dee641886fTorne (Richard Coles) 3755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def test_rebaseline_all(self): 3763c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self._setup_mock_builder_data() 3773c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch 378e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch options = MockOptions(optimize=True, verbose=True, results_directory=None) 3793c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self._write("userscripts/first-test.html", "Dummy test contents") 3803c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self.command._rebaseline(options, {"userscripts/first-test.html": {"MOCK builder": ["txt", "png"]}}) 3815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Note that we have one run_in_parallel() call followed by a run_command() 383926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(self.tool.executive.calls, 384c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) [[['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']], 385c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']], 386c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'txt,png', 'userscripts/first-test.html', '--verbose']]]) 3875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def test_rebaseline_debug(self): 3893c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self._setup_mock_builder_data() 3903c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch 391e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch options = MockOptions(optimize=True, verbose=True, results_directory=None) 3923c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self._write("userscripts/first-test.html", "Dummy test contents") 3933c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self.command._rebaseline(options, {"userscripts/first-test.html": {"MOCK builder (Debug)": ["txt", "png"]}}) 3945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Note that we have one run_in_parallel() call followed by a run_command() 396926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(self.tool.executive.calls, 397c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) [[['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.html', '--verbose']], 398c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.html', '--verbose']], 399c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'txt,png', 'userscripts/first-test.html', '--verbose']]]) 4005c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4015c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def test_no_optimize(self): 4023c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self._setup_mock_builder_data() 4033c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch 404e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch options = MockOptions(optimize=False, verbose=True, results_directory=None) 4053c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self._write("userscripts/first-test.html", "Dummy test contents") 4063c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self.command._rebaseline(options, {"userscripts/first-test.html": {"MOCK builder (Debug)": ["txt", "png"]}}) 4075c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4085c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Note that we have only one run_in_parallel() call 409926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(self.tool.executive.calls, 410c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) [[['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.html', '--verbose']], 411c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder (Debug)', '--test', 'userscripts/first-test.html', '--verbose']]]) 4125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def test_results_directory(self): 4143c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self._setup_mock_builder_data() 4153c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch 416e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch options = MockOptions(optimize=False, verbose=True, results_directory='/tmp') 4173c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self._write("userscripts/first-test.html", "Dummy test contents") 4183c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self.command._rebaseline(options, {"userscripts/first-test.html": {"MOCK builder": ["txt", "png"]}}) 4195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 4205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # Note that we have only one run_in_parallel() call 421926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(self.tool.executive.calls, 422c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) [[['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--results-directory', '/tmp', '--verbose']], 423c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--results-directory', '/tmp', '--verbose']]]) 4245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 42553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles)class TestRebaselineJsonUpdatesExpectationsFiles(_BaseTestCase): 42653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) command_constructor = RebaselineJson 42753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 42853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def setUp(self): 42953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) super(TestRebaselineJsonUpdatesExpectationsFiles, self).setUp() 43053e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self.tool.executive = MockExecutive2() 43153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 43253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def mock_run_command(args, 43353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) cwd=None, 43453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) input=None, 43553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) error_handler=None, 43653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) return_exit_code=False, 43753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) return_stderr=True, 43853e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) decode_output=False, 43953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) env=None): 4403c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch return '{"add": [], "remove-lines": [{"test": "userscripts/first-test.html", "builder": "WebKit Mac10.7"}]}\n' 44153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) self.tool.executive.run_command = mock_run_command 44253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 44353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def test_rebaseline_updates_expectations_file(self): 444e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch options = MockOptions(optimize=False, verbose=True, results_directory=None) 44553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 4463c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self._write(self.lion_expectations_path, "Bug(x) [ Mac ] userscripts/first-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/first-test.html [ ImageOnlyFailure ]\n") 4473c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self._write("userscripts/first-test.html", "Dummy test contents") 4483c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self._setup_mock_builder_data() 44953e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 4503c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self.command._rebaseline(options, {"userscripts/first-test.html": {"WebKit Mac10.7": ["txt", "png"]}}) 45153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 45253e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) new_expectations = self._read(self.lion_expectations_path) 45309380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) self.assertMultiLineEqual(new_expectations, "Bug(x) [ Mavericks MountainLion Retina SnowLeopard ] userscripts/first-test.html [ ImageOnlyFailure ]\nbug(z) [ Linux ] userscripts/first-test.html [ ImageOnlyFailure ]\n") 45453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 45553e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) def test_rebaseline_updates_expectations_file_all_platforms(self): 456e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch options = MockOptions(optimize=False, verbose=True, results_directory=None) 45753e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 4583c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self._write(self.lion_expectations_path, "Bug(x) userscripts/first-test.html [ ImageOnlyFailure ]\n") 4593c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self._write("userscripts/first-test.html", "Dummy test contents") 4603c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self._setup_mock_builder_data() 46153e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 4623c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self.command._rebaseline(options, {"userscripts/first-test.html": {"WebKit Mac10.7": ["txt", "png"]}}) 46353e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 46453e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) new_expectations = self._read(self.lion_expectations_path) 46509380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) self.assertMultiLineEqual(new_expectations, "Bug(x) [ Android Linux Mavericks MountainLion Retina SnowLeopard Win ] userscripts/first-test.html [ ImageOnlyFailure ]\n") 46653e740f4a82e17f3ae59772501622dc354e42336Torne (Richard Coles) 467c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) def test_rebaseline_handles_platform_skips(self): 468c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # This test is just like test_rebaseline_updates_expectations_file_all_platforms(), 469c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # except that if a particular port happens to SKIP a test in an overrides file, 470c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # we count that as passing, and do not think that we still need to rebaseline it. 471c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) options = MockOptions(optimize=False, verbose=True, results_directory=None) 472c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) 473c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) self._write(self.lion_expectations_path, "Bug(x) userscripts/first-test.html [ ImageOnlyFailure ]\n") 4749bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) self._write("NeverFixTests", "Bug(y) [ Android ] userscripts [ Skip ]\n") 475c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) self._write("userscripts/first-test.html", "Dummy test contents") 476c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) self._setup_mock_builder_data() 477c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) 478c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) self.command._rebaseline(options, {"userscripts/first-test.html": {"WebKit Mac10.7": ["txt", "png"]}}) 479c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) 480c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) new_expectations = self._read(self.lion_expectations_path) 48109380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) self.assertMultiLineEqual(new_expectations, "Bug(x) [ Linux Mavericks MountainLion Retina SnowLeopard Win ] userscripts/first-test.html [ ImageOnlyFailure ]\n") 482c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) 483c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) def test_rebaseline_handles_skips_in_file(self): 484c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # This test is like test_Rebaseline_handles_platform_skips, except that the 485c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # Skip is in the same (generic) file rather than a platform file. In this case, 486c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # the Skip line should be left unmodified. Note that the first line is now 487c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # qualified as "[Linux Mac Win]"; if it was unqualified, it would conflict with 488c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) # the second line. 489c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) options = MockOptions(optimize=False, verbose=True, results_directory=None) 490c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) 491c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) self._write(self.lion_expectations_path, 492c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) ("Bug(x) [ Linux Mac Win ] userscripts/first-test.html [ ImageOnlyFailure ]\n" 493c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) "Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n")) 494c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) self._write("userscripts/first-test.html", "Dummy test contents") 495c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) self._setup_mock_builder_data() 496c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) 497c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) self.command._rebaseline(options, {"userscripts/first-test.html": {"WebKit Mac10.7": ["txt", "png"]}}) 498c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) 499c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) new_expectations = self._read(self.lion_expectations_path) 500c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) self.assertMultiLineEqual(new_expectations, 50109380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) ("Bug(x) [ Linux Mavericks MountainLion Retina SnowLeopard Win ] userscripts/first-test.html [ ImageOnlyFailure ]\n" 502c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) "Bug(y) [ Android ] userscripts/first-test.html [ Skip ]\n")) 503c0e19a689c8ac22cdc96b291a8d33a5d3b0b34a4Torne (Richard Coles) 5049bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) def test_rebaseline_handles_smoke_tests(self): 5059bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) # This test is just like test_rebaseline_handles_platform_skips, except that we check for 5069bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) # a test not being in the SmokeTests file, instead of using overrides files. 5079bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) # If a test is not part of the smoke tests, we count that as passing on ports that only 5089bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) # run smoke tests, and do not think that we still need to rebaseline it. 5099bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) options = MockOptions(optimize=False, verbose=True, results_directory=None) 5109bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) 5119bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) self._write(self.lion_expectations_path, "Bug(x) userscripts/first-test.html [ ImageOnlyFailure ]\n") 5129bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) self._write("SmokeTests", "fast/html/article-element.html") 5139bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) self._write("userscripts/first-test.html", "Dummy test contents") 5149bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) self._setup_mock_builder_data() 5159bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) 5169bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) self.command._rebaseline(options, {"userscripts/first-test.html": {"WebKit Mac10.7": ["txt", "png"]}}) 5179bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) 5189bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) new_expectations = self._read(self.lion_expectations_path) 51909380295ba73501a205346becac22c6978e4671dTorne (Richard Coles) self.assertMultiLineEqual(new_expectations, "Bug(x) [ Linux Mavericks MountainLion Retina SnowLeopard Win ] userscripts/first-test.html [ ImageOnlyFailure ]\n") 5209bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) 5219bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) 5225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class TestRebaseline(_BaseTestCase): 5235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # This command shares most of its logic with RebaselineJson, so these tests just test what is different. 5245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) command_constructor = Rebaseline # AKA webkit-patch rebaseline 5265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def test_rebaseline(self): 5285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.command._builders_to_pull_from = lambda: [MockBuilder('MOCK builder')] 5295c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5303c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self._write("userscripts/first-test.html", "test data") 53181a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) 5325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._zero_out_test_expectations() 5333c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self._setup_mock_builder_data() 5345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) old_exact_matches = builders._exact_matches 5365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) try: 5375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) builders._exact_matches = { 5385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) "MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 5395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) } 5403c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self.command.execute(MockOptions(results_directory=False, optimize=False, builders=None, suffixes="txt,png", verbose=True), ['userscripts/first-test.html'], self.tool) 5415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) finally: 5425c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) builders._exact_matches = old_exact_matches 5435c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5445c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) calls = filter(lambda x: x != ['qmake', '-v'] and x[0] != 'perl', self.tool.executive.calls) 545926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(calls, 546c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) [[['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']], 547c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose']]]) 5485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 54981a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) def test_rebaseline_directory(self): 55081a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) self.command._builders_to_pull_from = lambda: [MockBuilder('MOCK builder')] 55181a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) 55281a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) self._write("userscripts/first-test.html", "test data") 55381a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) self._write("userscripts/second-test.html", "test data") 55481a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) 5553c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self._setup_mock_builder_data() 5563c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch 55781a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) old_exact_matches = builders._exact_matches 55881a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) try: 55981a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) builders._exact_matches = { 56081a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) "MOCK builder": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 56181a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) } 562e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.command.execute(MockOptions(results_directory=False, optimize=False, builders=None, suffixes="txt,png", verbose=True), ['userscripts'], self.tool) 56381a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) finally: 56481a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) builders._exact_matches = old_exact_matches 56581a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) 56681a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) calls = filter(lambda x: x != ['qmake', '-v'] and x[0] != 'perl', self.tool.executive.calls) 56781a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) self.assertEqual(calls, 568c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) [[['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose'], 569c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/second-test.html', '--verbose']], 570c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) [['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/first-test.html', '--verbose'], 571c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK builder', '--test', 'userscripts/second-test.html', '--verbose']]]) 57281a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) 5735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 574f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles)class MockLineRemovingExecutive(MockExecutive): 575f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) def run_in_parallel(self, commands): 576f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) assert len(commands) 577f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 578f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) num_previous_calls = len(self.calls) 579f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) command_outputs = [] 580f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) for cmd_line, cwd in commands: 581f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) out = self.run_command(cmd_line, cwd=cwd) 582f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) if 'rebaseline-test-internal' in cmd_line: 583c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) out = '{"add": [], "remove-lines": [{"test": "%s", "builder": "%s"}], "delete": []}\n' % (cmd_line[8], cmd_line[6]) 584f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) command_outputs.append([0, out, '']) 585f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 586f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) new_calls = self.calls[num_previous_calls:] 587f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.calls = self.calls[:num_previous_calls] 588f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.calls.append(new_calls) 589f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) return command_outputs 590f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 591f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 5925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class TestRebaselineExpectations(_BaseTestCase): 5935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) command_constructor = RebaselineExpectations 5945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 5955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def setUp(self): 5965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) super(TestRebaselineExpectations, self).setUp() 597e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.options = MockOptions(optimize=False, builders=None, suffixes=['txt'], verbose=False, platform=None, results_directory=None) 5985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 599f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) def _write_test_file(self, port, path, contents): 600f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) abs_path = self.tool.filesystem.join(port.layout_tests_dir(), path) 601f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.tool.filesystem.write_text_file(abs_path, contents) 602f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 603f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) def _setup_test_port(self): 604f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) test_port = self.tool.port_factory.get('test') 605f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) original_get = self.tool.port_factory.get 606f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 607f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) def get_test_port(port_name=None, options=None, **kwargs): 608f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) if not port_name: 609f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) return test_port 610f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) return original_get(port_name, options, **kwargs) 611f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) # Need to make sure all the ports grabbed use the test checkout path instead of the mock checkout path. 612f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) # FIXME: crbug.com/279494 - we shouldn't be doing this. 613f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.tool.port_factory.get = get_test_port 614f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 615f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) return test_port 616f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 6175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def test_rebaseline_expectations(self): 6185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._zero_out_test_expectations() 6195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 6205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.tool.executive = MockExecutive2() 6215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 6223c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch def builder_data(): 6233c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self.command._builder_data['MOCK SnowLeopard'] = self.command._builder_data['MOCK Leopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({ 6243c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch "tests": { 6253c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch "userscripts": { 6263c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch "another-test.html": { 6273c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch "expected": "PASS", 6283c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch "actual": "PASS TEXT" 6293c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch }, 6303c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch "images.svg": { 6313c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch "expected": "FAIL", 6323c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch "actual": "IMAGE+TEXT" 6333c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch } 6343c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch } 6353c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch } 6363c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch});""") 6373c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch return self.command._builder_data 6383c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch 6393c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self.command.builder_data = builder_data 6403c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch 64181a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) self._write("userscripts/another-test.html", "Dummy test contents") 64281a5157921f1d2a7ff6aae115bfe3c139b38a5c8Torne (Richard Coles) self._write("userscripts/images.svg", "Dummy test contents") 6433c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self.command._tests_to_rebaseline = lambda port: { 6443c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch 'userscripts/another-test.html': set(['txt']), 6453c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch 'userscripts/images.svg': set(['png']), 6463c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch 'userscripts/not-actually-failing.html': set(['txt', 'png', 'wav']), 6473c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch } 6483c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch 6493c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch old_exact_matches = builders._exact_matches 6503c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch try: 6513c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch builders._exact_matches = { 6523c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 6533c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 6543c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch } 6553c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self.command.execute(self.options, [], self.tool) 6563c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch finally: 6573c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch builders._exact_matches = old_exact_matches 6585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 6595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # FIXME: change this to use the test- ports. 6605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) calls = filter(lambda x: x != ['qmake', '-v'], self.tool.executive.calls) 6613c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch self.assertEqual(self.tool.executive.calls, [ 6623c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch [ 663c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'userscripts/another-test.html'], 664c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/another-test.html'], 665c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'png', '--builder', 'MOCK Leopard', '--test', 'userscripts/images.svg'], 666c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'png', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/images.svg'], 6673c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch ], 6683c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch [ 669c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'userscripts/another-test.html'], 670c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/another-test.html'], 671c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'MOCK Leopard', '--test', 'userscripts/images.svg'], 672c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'MOCK SnowLeopard', '--test', 'userscripts/images.svg'], 673c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ], 6743c9e4aeaee9f9b0a9a814da07bcb33319c7ea363Ben Murdoch ]) 6755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 6765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def test_rebaseline_expectations_noop(self): 6775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._zero_out_test_expectations() 6785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 6795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) oc = OutputCapture() 6805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) try: 6815c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) oc.capture_output() 6825c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.command.execute(self.options, [], self.tool) 6835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) finally: 6845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) _, _, logs = oc.restore_output() 685926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(self.tool.filesystem.written_files, {}) 686926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(logs, 'Did not find any tests marked Rebaseline.\n') 6875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 6885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def disabled_test_overrides_are_included_correctly(self): 6895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # This tests that the any tests marked as REBASELINE in the overrides are found, but 6905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) # that the overrides do not get written into the main file. 6915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._zero_out_test_expectations() 6925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 6935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._write(self.lion_expectations_path, '') 6945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.lion_port.expectations_dict = lambda: { 6955c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.lion_expectations_path: '', 6965c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'overrides': ('Bug(x) userscripts/another-test.html [ Failure Rebaseline ]\n' 6975c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'Bug(y) userscripts/test.html [ Crash ]\n')} 6985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._write('/userscripts/another-test.html', '') 6995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 700926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertDictEqual(self.command._tests_to_rebaseline(self.lion_port), {'userscripts/another-test.html': set(['png', 'txt', 'wav'])}) 701926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(self._read(self.lion_expectations_path), '') 7025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 70393ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) def test_rebaseline_without_other_expectations(self): 70493ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) self._write("userscripts/another-test.html", "Dummy test contents") 70593ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) self._write(self.lion_expectations_path, "Bug(x) userscripts/another-test.html [ Rebaseline ]\n") 70693ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) self.assertDictEqual(self.command._tests_to_rebaseline(self.lion_port), {'userscripts/another-test.html': ('png', 'wav', 'txt')}) 70793ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) 708f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) def test_rebaseline_test_passes_everywhere(self): 709f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) test_port = self._setup_test_port() 710f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 711f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) old_builder_data = self.command.builder_data 712f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 713f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) def builder_data(): 714f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.command._builder_data['MOCK Leopard'] = self.command._builder_data['MOCK SnowLeopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({ 715f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) "tests": { 716f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) "fast": { 717f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) "dom": { 718f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) "prototype-taco.html": { 719f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) "expected": "FAIL", 720f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) "actual": "PASS", 721f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) "is_unexpected": true 722f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) } 723f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) } 724f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) } 725f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) } 726f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles)});""") 727f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) return self.command._builder_data 728f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 729f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.command.builder_data = builder_data 730f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 731f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expectations_file(), """ 732f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles)Bug(foo) fast/dom/prototype-taco.html [ Rebaseline ] 733f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles)""") 734f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 735f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents") 736f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 737f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.tool.executive = MockLineRemovingExecutive() 738f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 739f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) old_exact_matches = builders._exact_matches 740f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) try: 741f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) builders._exact_matches = { 742f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 743f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 744f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) } 745f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 746f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.command.execute(self.options, [], self.tool) 747f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.assertEqual(self.tool.executive.calls, []) 748f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 749f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) # The mac ports should both be removed since they're the only ones in builders._exact_matches. 750f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.assertEqual(self.tool.filesystem.read_text_file(test_port.path_to_generic_test_expectations_file()), """ 751f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles)Bug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ Rebaseline ] 752f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles)""") 753f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) finally: 754f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) builders._exact_matches = old_exact_matches 755f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 7565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 7575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class _FakeOptimizer(BaselineOptimizer): 7585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def read_results_by_directory(self, baseline_name): 7595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if baseline_name.endswith('txt'): 76093ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) return {'LayoutTests/passes/text.html': '123456'} 7615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return {} 7625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 7635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 764f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles)class TestOptimizeBaselines(_BaseTestCase): 765f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) command_constructor = OptimizeBaselines 766f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 767f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) def _write_test_file(self, port, path, contents): 768f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) abs_path = self.tool.filesystem.join(port.layout_tests_dir(), path) 769f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.tool.filesystem.write_text_file(abs_path, contents) 770f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 771f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) def setUp(self): 772f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) super(TestOptimizeBaselines, self).setUp() 773f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 774f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) # FIXME: This is a hack to get the unittest and the BaselineOptimize to both use /mock-checkout 775f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) # instead of one using /mock-checkout and one using /test-checkout. 776f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) default_port = self.tool.port_factory.get() 777f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.tool.port_factory.get = lambda port_name=None: default_port 778f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 7797242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci def test_modify_scm(self): 780f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) test_port = self.tool.port_factory.get('test') 781f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self._write_test_file(test_port, 'another/test.html', "Dummy test contents") 782f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self._write_test_file(test_port, 'platform/mac-snowleopard/another/test-expected.txt', "result A") 783f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self._write_test_file(test_port, 'another/test-expected.txt', "result A") 784f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 785f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) old_exact_matches = builders._exact_matches 786f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) try: 787f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) builders._exact_matches = { 788f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) "MOCK Leopard Debug": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 789f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) } 790f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) OutputCapture().assert_outputs(self, self.command.execute, args=[ 791f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) MockOptions(suffixes='txt', no_modify_scm=False, platform='test-mac-snowleopard'), 792f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) ['another/test.html'], 793f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.tool, 794f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) ], expected_stdout='{"add": [], "remove-lines": [], "delete": []}\n') 795f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) finally: 796f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) builders._exact_matches = old_exact_matches 797f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 798f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(test_port.layout_tests_dir(), 'platform/mac/another/test-expected.txt'))) 799f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.assertTrue(self.tool.filesystem.exists(self.tool.filesystem.join(test_port.layout_tests_dir(), 'another/test-expected.txt'))) 800f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 801f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) def test_no_modify_scm(self): 802f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) test_port = self.tool.port_factory.get('test') 803f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self._write_test_file(test_port, 'another/test.html', "Dummy test contents") 804f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self._write_test_file(test_port, 'platform/mac-snowleopard/another/test-expected.txt', "result A") 805f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self._write_test_file(test_port, 'another/test-expected.txt', "result A") 806f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 807f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) old_exact_matches = builders._exact_matches 808f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) try: 809f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) builders._exact_matches = { 810f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) "MOCK Leopard Debug": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 811f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) } 812f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) OutputCapture().assert_outputs(self, self.command.execute, args=[ 813f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) MockOptions(suffixes='txt', no_modify_scm=True, platform='test-mac-snowleopard'), 814f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) ['another/test.html'], 815f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.tool, 816f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) ], expected_stdout='{"add": [], "remove-lines": [], "delete": ["/mock-checkout/third_party/WebKit/LayoutTests/platform/mac-snowleopard/another/test-expected.txt"]}\n') 817f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) finally: 818f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) builders._exact_matches = old_exact_matches 819f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 820f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(test_port.layout_tests_dir(), 'platform/mac/another/test-expected.txt'))) 821f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.assertTrue(self.tool.filesystem.exists(self.tool.filesystem.join(test_port.layout_tests_dir(), 'another/test-expected.txt'))) 822f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 8237242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci def test_optimize_all_suffixes_by_default(self): 8247242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci test_port = self.tool.port_factory.get('test') 8257242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci self._write_test_file(test_port, 'another/test.html', "Dummy test contents") 8267242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci self._write_test_file(test_port, 'platform/mac-snowleopard/another/test-expected.txt', "result A") 8277242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci self._write_test_file(test_port, 'platform/mac-snowleopard/another/test-expected.png', "result A png") 8287242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci self._write_test_file(test_port, 'another/test-expected.txt', "result A") 8297242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci self._write_test_file(test_port, 'another/test-expected.png', "result A png") 8307242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci 8317242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci old_exact_matches = builders._exact_matches 8327242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci try: 8337242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci builders._exact_matches = { 8347242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci "MOCK Leopard Debug": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 8357242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci } 8367242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci oc = OutputCapture() 8377242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci oc.capture_output() 8387242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci self.command.execute(MockOptions(suffixes='txt,wav,png', no_modify_scm=True, platform='test-mac-snowleopard'), 8397242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci ['another/test.html'], 8407242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci self.tool) 8417242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci finally: 8427242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci out, err, logs = oc.restore_output() 8437242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci builders._exact_matches = old_exact_matches 8447242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci 8457242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci self.assertEquals(out, '{"add": [], "remove-lines": [], "delete": ["/mock-checkout/third_party/WebKit/LayoutTests/platform/mac-snowleopard/another/test-expected.txt", "/mock-checkout/third_party/WebKit/LayoutTests/platform/mac-snowleopard/another/test-expected.png"]}\n') 8467242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(test_port.layout_tests_dir(), 'platform/mac/another/test-expected.txt'))) 8477242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci self.assertFalse(self.tool.filesystem.exists(self.tool.filesystem.join(test_port.layout_tests_dir(), 'platform/mac/another/test-expected.png'))) 8487242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci self.assertTrue(self.tool.filesystem.exists(self.tool.filesystem.join(test_port.layout_tests_dir(), 'another/test-expected.txt'))) 8497242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci self.assertTrue(self.tool.filesystem.exists(self.tool.filesystem.join(test_port.layout_tests_dir(), 'another/test-expected.png'))) 8507242dc3dbeb210b5e876a3c42d1ec1a667fc621aPrimiano Tucci 851f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 8525c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class TestAnalyzeBaselines(_BaseTestCase): 8535c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) command_constructor = AnalyzeBaselines 8545c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 8555c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def setUp(self): 8565c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) super(TestAnalyzeBaselines, self).setUp() 8575c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.port = self.tool.port_factory.get('test') 8585c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.tool.port_factory.get = (lambda port_name=None, options=None: self.port) 8595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.lines = [] 8605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.command._optimizer_class = _FakeOptimizer 861926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.command._write = (lambda msg: self.lines.append(msg)) # pylint bug warning about unnecessary lambda? pylint: disable=W0108 8625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 8635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def test_default(self): 8645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.command.execute(MockOptions(suffixes='txt', missing=False, platform=None), ['passes/text.html'], self.tool) 865926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(self.lines, 8665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ['passes/text-expected.txt:', 86793ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) ' (generic): 123456']) 8685c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 8695c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def test_missing_baselines(self): 8705c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self.command.execute(MockOptions(suffixes='png,txt', missing=True, platform=None), ['passes/text.html'], self.tool) 871926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertEqual(self.lines, 8725c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ['passes/text-expected.png: (no baselines found)', 8735c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 'passes/text-expected.txt:', 87493ac45cfc74041c8ae536ce58a9534d46db2024eTorne (Richard Coles) ' (generic): 123456']) 87583750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 87683750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 87783750176c3ee2cea66c8a9751271026a5901be3aBen Murdochclass TestAutoRebaseline(_BaseTestCase): 87883750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch command_constructor = AutoRebaseline 87983750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 88083750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch def _write_test_file(self, port, path, contents): 88183750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch abs_path = self.tool.filesystem.join(port.layout_tests_dir(), path) 88283750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch self.tool.filesystem.write_text_file(abs_path, contents) 88383750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 884d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) def _setup_test_port(self): 885d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) test_port = self.tool.port_factory.get('test') 886d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) original_get = self.tool.port_factory.get 887d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) 888d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) def get_test_port(port_name=None, options=None, **kwargs): 889d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) if not port_name: 890d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) return test_port 891d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) return original_get(port_name, options, **kwargs) 892d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) # Need to make sure all the ports grabbed use the test checkout path instead of the mock checkout path. 893d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) # FIXME: crbug.com/279494 - we shouldn't be doing this. 894d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) self.tool.port_factory.get = get_test_port 895d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) 896d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) return test_port 897d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) 89883750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch def setUp(self): 89983750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch super(TestAutoRebaseline, self).setUp() 9005d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) self.command.latest_revision_processed_on_all_bots = lambda: 9000 9015d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) self.command.bot_revision_data = lambda: [{"builder": "Mock builder", "revision": "9000"}] 90283750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 903d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) def test_release_builders(self): 904d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) old_exact_matches = builders._exact_matches 905d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) try: 906d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) builders._exact_matches = { 907d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 908d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) "MOCK Leopard Debug": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 909d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) "MOCK Leopard ASAN": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 910d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) } 911d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) self.assertEqual(self.command._release_builders(), ['MOCK Leopard']) 912d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) finally: 913d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) builders._exact_matches = old_exact_matches 914d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) 91583750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch def test_tests_to_rebaseline(self): 91683750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch def blame(path): 91783750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch return """ 91883750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch624c3081c0 path/to/TestExpectations (foobarbaz1@chromium.org 2013-06-14 20:18:46 +0000 11) crbug.com/24182 [ Debug ] path/to/norebaseline.html [ ImageOnlyFailure ] 91983750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch624c3081c0 path/to/TestExpectations (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000 13) Bug(foo) path/to/rebaseline-without-bug-number.html [ NeedsRebaseline ] 92083750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch624c3081c0 path/to/TestExpectations (foobarbaz1@chromium.org 2013-06-14 20:18:46 +0000 11) crbug.com/24182 [ Debug ] path/to/rebaseline-with-modifiers.html [ NeedsRebaseline ] 92183750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch624c3081c0 path/to/TestExpectations (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000 12) crbug.com/24182 crbug.com/234 path/to/rebaseline-without-modifiers.html [ NeedsRebaseline ] 92283750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch6469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/rebaseline-new-revision.html [ NeedsRebaseline ] 92383750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch624caaaaaa path/to/TestExpectations (foo@chromium.org 2013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ] 92483750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch0000000000 path/to/TestExpectations (foo@chromium.org 2013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ] 92583750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch""" 92683750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch self.tool.scm().blame = blame 92783750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 92883750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch min_revision = 9000 9295d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) self.assertEqual(self.command.tests_to_rebaseline(self.tool, min_revision, print_revisions=False), ( 93083750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch set(['path/to/rebaseline-without-bug-number.html', 'path/to/rebaseline-with-modifiers.html', 'path/to/rebaseline-without-modifiers.html']), 93183750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 5678, 93283750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 'foobarbaz1@chromium.org', 933e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) set(['24182', '234']), 934e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) True)) 93583750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 93683750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch def test_tests_to_rebaseline_over_limit(self): 93783750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch def blame(path): 93883750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch result = "" 93983750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch for i in range(0, self.command.MAX_LINES_TO_REBASELINE + 1): 94083750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch result += "624c3081c0 path/to/TestExpectations (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000 13) crbug.com/24182 path/to/rebaseline-%s.html [ NeedsRebaseline ]\n" % i 94183750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch return result 94283750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch self.tool.scm().blame = blame 94383750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 94483750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch expected_list_of_tests = [] 94583750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch for i in range(0, self.command.MAX_LINES_TO_REBASELINE): 94683750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch expected_list_of_tests.append("path/to/rebaseline-%s.html" % i) 94783750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 94883750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch min_revision = 9000 9495d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) self.assertEqual(self.command.tests_to_rebaseline(self.tool, min_revision, print_revisions=False), ( 95083750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch set(expected_list_of_tests), 95183750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 5678, 95283750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 'foobarbaz1@chromium.org', 953e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) set(['24182']), 954e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) True)) 95583750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 95683750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch def test_commit_message(self): 95783750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch author = "foo@chromium.org" 95883750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch revision = 1234 95983750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch bugs = set() 96083750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch self.assertEqual(self.command.commit_message(author, revision, bugs), 96183750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch """Auto-rebaseline for r1234 96283750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 96383750176c3ee2cea66c8a9751271026a5901be3aBen Murdochhttp://src.chromium.org/viewvc/blink?view=revision&revision=1234 96483750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 96583750176c3ee2cea66c8a9751271026a5901be3aBen MurdochTBR=foo@chromium.org 96683750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch""") 96783750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 96883750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch bugs = set(["234", "345"]) 96983750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch self.assertEqual(self.command.commit_message(author, revision, bugs), 97083750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch """Auto-rebaseline for r1234 97183750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 97283750176c3ee2cea66c8a9751271026a5901be3aBen Murdochhttp://src.chromium.org/viewvc/blink?view=revision&revision=1234 97383750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 97483750176c3ee2cea66c8a9751271026a5901be3aBen MurdochBUG=234,345 97583750176c3ee2cea66c8a9751271026a5901be3aBen MurdochTBR=foo@chromium.org 97683750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch""") 97783750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 97883750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch def test_no_needs_rebaseline_lines(self): 97983750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch def blame(path): 98083750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch return """ 98183750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch6469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2013-06-14 20:18:46 +0000 11) crbug.com/24182 [ Debug ] path/to/norebaseline.html [ ImageOnlyFailure ] 98283750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch""" 98383750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch self.tool.scm().blame = blame 98483750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 9855d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) self.command.execute(MockOptions(optimize=True, verbose=False, move_overwritten_baselines=False, results_directory=False), [], self.tool) 98683750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch self.assertEqual(self.tool.executive.calls, []) 98783750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 98883750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch def test_execute(self): 98983750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch def blame(path): 99083750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch return """ 991f79f16f17ddc4f842d7b7a38603e280e94be826aTorne (Richard Coles)6469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2013-06-14 20:18:46 +0000 11) # Test NeedsRebaseline being in a comment doesn't bork parsing. 99283750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch6469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2013-06-14 20:18:46 +0000 11) crbug.com/24182 [ Debug ] path/to/norebaseline.html [ ImageOnlyFailure ] 993fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch6469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] 994fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch6469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2013-06-14 20:18:46 +0000 11) crbug.com/24182 [ SnowLeopard ] fast/dom/prototype-strawberry.html [ NeedsRebaseline ] 995fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch6469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000 12) crbug.com/24182 fast/dom/prototype-chocolate.html [ NeedsRebaseline ] 99683750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch624caaaaaa path/to/TestExpectations (foo@chromium.org 2013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ] 99783750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch0000000000 path/to/TestExpectations (foo@chromium.org 2013-04-28 04:52:41 +0000 12) crbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ] 99883750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch""" 99983750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch self.tool.scm().blame = blame 100083750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 1001d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) test_port = self._setup_test_port() 100283750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 1003fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch old_builder_data = self.command.builder_data 1004fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch 1005fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch def builder_data(): 1006fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch old_builder_data() 1007fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch # have prototype-chocolate only fail on "MOCK Leopard". 1008fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch self.command._builder_data['MOCK SnowLeopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({ 1009fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "tests": { 1010fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "fast": { 1011fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "dom": { 1012fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "prototype-taco.html": { 1013fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "expected": "PASS", 1014fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "actual": "PASS TEXT", 1015fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "is_unexpected": true 1016fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch }, 1017fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "prototype-chocolate.html": { 1018fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "expected": "FAIL", 1019fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "actual": "PASS" 1020fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch }, 1021fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "prototype-strawberry.html": { 1022fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "expected": "PASS", 1023fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "actual": "IMAGE PASS", 1024fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "is_unexpected": true 1025fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch } 1026fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch } 1027fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch } 1028fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch } 1029fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch});""") 1030fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch return self.command._builder_data 1031fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch 1032fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch self.command.builder_data = builder_data 1033fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch 103483750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expectations_file(), """ 103583750176c3ee2cea66c8a9751271026a5901be3aBen Murdochcrbug.com/24182 [ Debug ] path/to/norebaseline.html [ Rebaseline ] 1036fff8884795cb540f87cf6e6d67b629519b00eb8bBen MurdochBug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] 1037fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdochcrbug.com/24182 [ SnowLeopard ] fast/dom/prototype-strawberry.html [ NeedsRebaseline ] 1038fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdochcrbug.com/24182 fast/dom/prototype-chocolate.html [ NeedsRebaseline ] 103983750176c3ee2cea66c8a9751271026a5901be3aBen Murdochcrbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ] 104083750176c3ee2cea66c8a9751271026a5901be3aBen Murdochcrbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ] 104183750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch""") 104283750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 1043fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents") 1044fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch self._write_test_file(test_port, 'fast/dom/prototype-strawberry.html', "Dummy test contents") 1045fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch self._write_test_file(test_port, 'fast/dom/prototype-chocolate.html', "Dummy test contents") 104683750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 1047f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.tool.executive = MockLineRemovingExecutive() 1048f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 104983750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch old_exact_matches = builders._exact_matches 105083750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch try: 105183750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch builders._exact_matches = { 105283750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 105383750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 105483750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch } 105583750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 105683750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch self.command.tree_status = lambda: 'closed' 10575d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) self.command.execute(MockOptions(optimize=True, verbose=False, move_overwritten_baselines=False, results_directory=False), [], self.tool) 105883750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch self.assertEqual(self.tool.executive.calls, []) 105983750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch 106083750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch self.command.tree_status = lambda: 'open' 106183750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch self.tool.executive.calls = [] 10625d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) self.command.execute(MockOptions(optimize=True, verbose=False, move_overwritten_baselines=False, results_directory=False), [], self.tool) 1063d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) 106483750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch self.assertEqual(self.tool.executive.calls, [ 106583750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch [ 1066c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt,png', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-chocolate.html'], 1067c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'png', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-strawberry.html'], 1068c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-taco.html'], 1069c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-taco.html'], 107083750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch ], 107183750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch [ 1072c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt,png', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-chocolate.html'], 1073c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'png', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-strawberry.html'], 1074c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK Leopard', '--test', 'fast/dom/prototype-taco.html'], 1075c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-taco.html'], 107683750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch ], 1077f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) [ 1078c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'txt,png', 'fast/dom/prototype-chocolate.html'], 1079c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'png', 'fast/dom/prototype-strawberry.html'], 1080c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'txt', 'fast/dom/prototype-taco.html'], 1081f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) ], 1082d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) ['git', 'cl', 'upload', '-f'], 10833464d02a173573db42f8ee6bb07bb74fabf4f5f2Ben Murdoch ['git', 'pull'], 1084d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) ['git', 'cl', 'dcommit', '-f'], 1085d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) ['git', 'cl', 'set_close'], 108683750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch ]) 1087fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch 1088fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch # The mac ports should both be removed since they're the only ones in builders._exact_matches. 1089fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch self.assertEqual(self.tool.filesystem.read_text_file(test_port.path_to_generic_test_expectations_file()), """ 1090fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdochcrbug.com/24182 [ Debug ] path/to/norebaseline.html [ Rebaseline ] 1091fff8884795cb540f87cf6e6d67b629519b00eb8bBen MurdochBug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ NeedsRebaseline ] 1092fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdochcrbug.com/24182 [ Linux Win ] fast/dom/prototype-chocolate.html [ NeedsRebaseline ] 1093fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdochcrbug.com/24182 path/to/not-cycled-through-bots.html [ NeedsRebaseline ] 1094fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdochcrbug.com/24182 path/to/locally-changed-lined.html [ NeedsRebaseline ] 1095fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch""") 1096fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch finally: 1097fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch builders._exact_matches = old_exact_matches 1098fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch 1099d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) def test_execute_git_cl_hangs(self): 1100fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch def blame(path): 1101fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch return """ 1102fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch6469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] 1103fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch""" 1104fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch self.tool.scm().blame = blame 1105fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch 1106d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) test_port = self._setup_test_port() 1107fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch 1108d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) old_builder_data = self.command.builder_data 1109d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) 1110d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) def builder_data(): 1111d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) old_builder_data() 1112d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) # have prototype-chocolate only fail on "MOCK Leopard". 1113d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) self.command._builder_data['MOCK SnowLeopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({ 1114d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) "tests": { 1115d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) "fast": { 1116d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) "dom": { 1117d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) "prototype-taco.html": { 1118d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) "expected": "PASS", 1119d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) "actual": "PASS TEXT", 1120d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) "is_unexpected": true 1121d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) } 1122d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) } 1123d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) } 1124d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) } 1125d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles)});""") 1126d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) return self.command._builder_data 1127d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) 1128d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) self.command.builder_data = builder_data 1129d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) 1130d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expectations_file(), """ 1131d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles)Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] 1132d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles)""") 1133d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) 1134d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents") 1135d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) 1136d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) old_exact_matches = builders._exact_matches 1137d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) try: 1138d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) builders._exact_matches = { 1139d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 1140d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) } 1141d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) 1142d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) self.command.SECONDS_BEFORE_GIVING_UP = 0 1143d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) self.command.tree_status = lambda: 'open' 1144d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) self.tool.executive.calls = [] 11455d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) self.command.execute(MockOptions(optimize=True, verbose=False, move_overwritten_baselines=False, results_directory=False), [], self.tool) 1146d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) 1147d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) self.assertEqual(self.tool.executive.calls, [ 1148d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) [ 1149c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'copy-existing-baselines-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-taco.html'], 1150d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) ], 1151d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) [ 1152c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) ['python', 'echo', 'rebaseline-test-internal', '--suffixes', 'txt', '--builder', 'MOCK SnowLeopard', '--test', 'fast/dom/prototype-taco.html'], 1153d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) ], 1154c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', 'txt', 'fast/dom/prototype-taco.html']], 1155d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) ['git', 'cl', 'upload', '-f'], 1156d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) ]) 1157d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) finally: 1158d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) builders._exact_matches = old_exact_matches 1159d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) 1160d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) def test_execute_test_passes_everywhere(self): 1161d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) def blame(path): 1162d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) return """ 1163d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles)6469e754a1 path/to/TestExpectations (foobarbaz1@chromium.org 2013-04-28 04:52:41 +0000 13) Bug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] 1164d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles)""" 1165d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) self.tool.scm().blame = blame 1166d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) 1167d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) test_port = self._setup_test_port() 1168fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch 1169fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch old_builder_data = self.command.builder_data 1170fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch 1171fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch def builder_data(): 1172fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch self.command._builder_data['MOCK Leopard'] = self.command._builder_data['MOCK SnowLeopard'] = LayoutTestResults.results_from_string("""ADD_RESULTS({ 1173fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "tests": { 1174fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "fast": { 1175fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "dom": { 1176fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "prototype-taco.html": { 1177f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) "expected": "FAIL", 1178f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) "actual": "PASS", 1179fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "is_unexpected": true 1180fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch } 1181fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch } 1182fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch } 1183fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch } 1184fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch});""") 1185fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch return self.command._builder_data 1186fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch 1187fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch self.command.builder_data = builder_data 1188fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch 1189fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch self.tool.filesystem.write_text_file(test_port.path_to_generic_test_expectations_file(), """ 1190fff8884795cb540f87cf6e6d67b629519b00eb8bBen MurdochBug(foo) fast/dom/prototype-taco.html [ NeedsRebaseline ] 1191fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch""") 1192fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch 1193fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch self._write_test_file(test_port, 'fast/dom/prototype-taco.html', "Dummy test contents") 1194fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch 1195f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) self.tool.executive = MockLineRemovingExecutive() 1196f6b7aed3f7ce69aca0d7a032d144cbd088b04393Torne (Richard Coles) 1197fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch old_exact_matches = builders._exact_matches 1198fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch try: 1199fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch builders._exact_matches = { 1200fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "MOCK Leopard": {"port_name": "test-mac-leopard", "specifiers": set(["mock-specifier"])}, 1201fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "MOCK SnowLeopard": {"port_name": "test-mac-snowleopard", "specifiers": set(["mock-specifier"])}, 1202fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch } 1203fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch 1204fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch self.command.tree_status = lambda: 'open' 12055d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) self.command.execute(MockOptions(optimize=True, verbose=False, move_overwritten_baselines=False, results_directory=False), [], self.tool) 1206fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch self.assertEqual(self.tool.executive.calls, [ 1207c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) [['python', 'echo', 'optimize-baselines', '--no-modify-scm', '--suffixes', '', 'fast/dom/prototype-taco.html']], 1208d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) ['git', 'cl', 'upload', '-f'], 1209fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch ['git', 'pull'], 1210d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) ['git', 'cl', 'dcommit', '-f'], 1211d6cdb82654e8f3343a693ca752d5c4cee0324e17Torne (Richard Coles) ['git', 'cl', 'set_close'], 1212fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch ]) 1213fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch 1214fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch # The mac ports should both be removed since they're the only ones in builders._exact_matches. 1215fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch self.assertEqual(self.tool.filesystem.read_text_file(test_port.path_to_generic_test_expectations_file()), """ 1216fff8884795cb540f87cf6e6d67b629519b00eb8bBen MurdochBug(foo) [ Linux Win ] fast/dom/prototype-taco.html [ NeedsRebaseline ] 1217fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch""") 121883750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch finally: 121983750176c3ee2cea66c8a9751271026a5901be3aBen Murdoch builders._exact_matches = old_exact_matches 12205d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) 12215d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) 12225d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles)class TestRebaselineOMatic(_BaseTestCase): 12235d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) command_constructor = RebaselineOMatic 12245d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) 12255d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) def setUp(self): 12265d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) super(TestRebaselineOMatic, self).setUp() 12275d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) self._logs = [] 12285d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) 1229c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) def _mock_log_to_server(self, log=''): 1230c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) self._logs.append(log) 12315d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) 12325d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) def test_run_logged_command(self): 12335d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) self.command._verbose = False 1234c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) self.command._post_log_to_server = self._mock_log_to_server 12355d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) self.command._run_logged_command(['echo', 'foo']) 12365d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) self.assertEqual(self.tool.executive.calls, [['echo', 'foo']]) 1237c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) self.assertEqual(self._logs, ['MOCK STDOUT']) 12385d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) 12395d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) def test_do_one_rebaseline(self): 12405d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) self.command._verbose = False 1241c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) self.command._post_log_to_server = self._mock_log_to_server 12425d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) 12435d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) oc = OutputCapture() 12445d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) oc.capture_output() 12455d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) self.command._do_one_rebaseline() 12465d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) out, _, _ = oc.restore_output() 12475d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) 12485d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) self.assertEqual(out, '') 12495d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) self.assertEqual(self.tool.executive.calls, [ 12505d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) ['git', 'pull'], 12515d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) ['/mock-checkout/third_party/WebKit/Tools/Scripts/webkit-patch', 'auto-rebaseline'], 12525d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) ]) 1253c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) self.assertEqual(self._logs, ['MOCK STDOUT']) 12545d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) 12555d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) def test_do_one_rebaseline_verbose(self): 12565d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) self.command._verbose = True 1257c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) self.command._post_log_to_server = self._mock_log_to_server 12585d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) 12595d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) oc = OutputCapture() 12605d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) oc.capture_output() 12615d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) self.command._do_one_rebaseline() 12625d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) out, _, _ = oc.restore_output() 12635d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) 12645d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) self.assertEqual(out, 'MOCK STDOUT\n') 12655d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) self.assertEqual(self.tool.executive.calls, [ 12665d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) ['git', 'pull'], 12675d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) ['/mock-checkout/third_party/WebKit/Tools/Scripts/webkit-patch', 'auto-rebaseline', '--verbose'], 12685d92fedcae5e801a8b224de090094f2d9df0b54aTorne (Richard Coles) ]) 1269c1847b1379d12d0e05df27436bf19a9b1bf12deaTorne (Richard Coles) self.assertEqual(self._logs, ['MOCK STDOUT']) 1270