15c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Copyright (c) 2010, Google Inc. All rights reserved. 25c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 35c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Redistribution and use in source and binary forms, with or without 45c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# modification, are permitted provided that the following conditions are 55c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# met: 65c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 75c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Redistributions of source code must retain the above copyright 85c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# notice, this list of conditions and the following disclaimer. 95c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Redistributions in binary form must reproduce the above 105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# copyright notice, this list of conditions and the following disclaimer 115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# in the documentation and/or other materials provided with the 125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# distribution. 135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Neither the name of Google Inc. nor the names of its 145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# contributors may be used to endorse or promote products derived from 155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# this software without specific prior written permission. 165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 29e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdochimport json 30926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)import logging 315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 32e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdochfrom webkitpy.common.memoized import memoized 33e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdochfrom webkitpy.layout_tests.layout_package import json_results_generator 34e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdochfrom webkitpy.layout_tests.models import test_expectations 35e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdochfrom webkitpy.layout_tests.models.test_expectations import TestExpectations 365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 37926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles)_log = logging.getLogger(__name__) 38926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 395c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 40e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch# These are helper functions for navigating the results json structure. 41e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdochdef for_each_test(tree, handler, prefix=''): 42e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch for key in tree: 43e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch new_prefix = (prefix + '/' + key) if prefix else key 44e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch if 'actual' not in tree[key]: 45e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch for_each_test(tree[key], handler, new_prefix) 46e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch else: 47e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch handler(new_prefix, tree[key]) 48e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 49e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 50e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdochdef result_for_test(tree, test): 51e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch parts = test.split('/') 52e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch for part in parts: 53fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch if part not in tree: 54fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch return None 55e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch tree = tree[part] 56e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch return tree 57e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 58e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 59e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdochclass JSONTestResult(object): 60e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def __init__(self, test_name, result_dict): 61e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self._test_name = test_name 62e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self._result_dict = result_dict 63e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 64e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def did_pass_or_run_as_expected(self): 65e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch return self.did_pass() or self.did_run_as_expected() 66e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 67e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def did_pass(self): 68e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch return test_expectations.PASS in self._actual_as_tokens() 69e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 70e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def did_run_as_expected(self): 71e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch return 'is_unexpected' not in self._result_dict 72e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 73e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def _tokenize(self, results_string): 74e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch tokens = map(TestExpectations.expectation_from_string, results_string.split(' ')) 75e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch if None in tokens: 76e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch _log.warning("Unrecognized result in %s" % results_string) 77e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch return set(tokens) 78e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch 79e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch @memoized 80e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def _actual_as_tokens(self): 81e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch actual_results = self._result_dict['actual'] 82e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch return self._tokenize(actual_results) 835c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 845c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 855c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# FIXME: This should be unified with ResultsSummary or other NRWT layout tests code 865c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# in the layout_tests package. 875c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# This doesn't belong in common.net, but we don't have a better place for it yet. 885c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class LayoutTestResults(object): 895c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) @classmethod 905c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def results_from_string(cls, string): 915c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if not string: 925c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return None 935c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 94e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch content_string = json_results_generator.strip_json_wrapper(string) 95e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch json_dict = json.loads(content_string) 96e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch if not json_dict: 97e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch return None 98e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch return cls(json_dict) 995c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 100e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def __init__(self, parsed_json): 101e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch self._results = parsed_json 1025c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 103e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def run_was_interrupted(self): 104e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch return self._results["interrupted"] 1055c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 106e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) def builder_name(self): 107e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) return self._results["builder_name"] 108e1f1df5f01594c0e62e751e4b46e779b85c2faa5Torne (Richard Coles) 109e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch def blink_revision(self): 110e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch return int(self._results["blink_revision"]) 111fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch 112fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch def actual_results(self, test): 113fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch result = result_for_test(self._results["tests"], test) 114fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch if result: 115fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch return result["actual"] 116fff8884795cb540f87cf6e6d67b629519b00eb8bBen Murdoch return "" 117