15c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Copyright (c) 2010, Google Inc. All rights reserved. 25c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 35c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Redistribution and use in source and binary forms, with or without 45c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# modification, are permitted provided that the following conditions are 55c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# met: 65c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 75c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Redistributions of source code must retain the above copyright 85c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# notice, this list of conditions and the following disclaimer. 95c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Redistributions in binary form must reproduce the above 105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# copyright notice, this list of conditions and the following disclaimer 115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# in the documentation and/or other materials provided with the 125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# distribution. 135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Neither the name of Google Inc. nor the names of its 145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# contributors may be used to endorse or promote products derived from 155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# this software without specific prior written permission. 165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 297757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochimport webkitpy.thirdparty.unittest2 as unittest 305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 31926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)from webkitpy.common.net.layouttestresults import LayoutTestResults 325c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.common.system.outputcapture import OutputCapture 335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.layout_tests.models import test_results 345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.layout_tests.models import test_failures 355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.thirdparty.BeautifulSoup import BeautifulSoup 365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 385c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class LayoutTestResultsTest(unittest.TestCase): 39e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch # The real files have no whitespace, but newlines make this much more readable. 40e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch example_full_results_json = """ADD_RESULTS({ 41e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "tests": { 42e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "fast": { 43e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "dom": { 44e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "prototype-inheritance.html": { 45e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "expected": "PASS", 46fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "actual": "TEXT", 47e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "is_unexpected": true 48e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch }, 49e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "prototype-banana.html": { 50e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "expected": "FAIL", 51e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "actual": "PASS", 52e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "is_unexpected": true 53e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch }, 54e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "prototype-taco.html": { 55e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "expected": "PASS", 56fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "actual": "PASS TEXT", 57e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "is_unexpected": true 58e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch }, 59e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "prototype-chocolate.html": { 60e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "expected": "FAIL", 61fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "actual": "IMAGE+TEXT" 62e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch }, 63e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "prototype-strawberry.html": { 64e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "expected": "PASS", 65fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch "actual": "IMAGE PASS", 66e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "is_unexpected": true 67e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch } 68e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch } 69e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch }, 70e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "svg": { 71e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "dynamic-updates": { 72e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "SVGFEDropShadowElement-dom-stdDeviation-attr.html": { 73e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "expected": "PASS", 74e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "actual": "IMAGE", 75e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "has_stderr": true, 76e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "is_unexpected": true 77e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch } 78e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch } 79e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch } 80e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch }, 81e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "skipped": 450, 82e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "num_regressions": 15, 83e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "layout_tests_dir": "\/b\/build\/slave\/Webkit_Mac10_5\/build\/src\/third_party\/WebKit\/LayoutTests", 84e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "version": 3, 85e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "num_passes": 77, 86e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "has_pretty_patch": false, 87e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "fixable": 1220, 88e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "num_flaky": 0, 89e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "blink_revision": "1234", 90e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch "has_wdiff": false 91e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch});""" 925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def test_results_from_string(self): 94926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertIsNone(LayoutTestResults.results_from_string(None)) 95926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) self.assertIsNone(LayoutTestResults.results_from_string("")) 96e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 97e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def test_was_interrupted(self): 98e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.assertTrue(LayoutTestResults.results_from_string('ADD_RESULTS({"tests":{},"interrupted":true});').run_was_interrupted()) 99e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.assertFalse(LayoutTestResults.results_from_string('ADD_RESULTS({"tests":{},"interrupted":false});').run_was_interrupted()) 100e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 101e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def test_blink_revision(self): 102e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self.assertEqual(LayoutTestResults.results_from_string(self.example_full_results_json).blink_revision(), 1234) 103fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch 104fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch def test_actual_results(self): 105fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch results = LayoutTestResults.results_from_string(self.example_full_results_json) 106fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch self.assertEqual(results.actual_results("fast/dom/prototype-banana.html"), "PASS") 107fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch self.assertEqual(results.actual_results("fast/dom/prototype-taco.html"), "PASS TEXT") 108fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch self.assertEqual(results.actual_results("nonexistant.html"), "") 109