19d33154b1362d89454297c21663b2403f1a75de5epoger@google.com#!/usr/bin/env python 29d33154b1362d89454297c21663b2403f1a75de5epoger@google.com# Copyright (c) 2013 The Chromium Authors. All rights reserved. 39d33154b1362d89454297c21663b2403f1a75de5epoger@google.com# Use of this source code is governed by a BSD-style license that can be 49d33154b1362d89454297c21663b2403f1a75de5epoger@google.com# found in the LICENSE file. 59d33154b1362d89454297c21663b2403f1a75de5epoger@google.com 69d33154b1362d89454297c21663b2403f1a75de5epoger@google.com"""Schema of the JSON summary file written out by the GM tool. 79d33154b1362d89454297c21663b2403f1a75de5epoger@google.com 89d33154b1362d89454297c21663b2403f1a75de5epoger@google.comThis must be kept in sync with the kJsonKey_ constants in gm_expectations.cpp ! 99d33154b1362d89454297c21663b2403f1a75de5epoger@google.com""" 109d33154b1362d89454297c21663b2403f1a75de5epoger@google.com 119d33154b1362d89454297c21663b2403f1a75de5epoger@google.com__author__ = 'Elliot Poger' 129d33154b1362d89454297c21663b2403f1a75de5epoger@google.com 139d33154b1362d89454297c21663b2403f1a75de5epoger@google.com 149d33154b1362d89454297c21663b2403f1a75de5epoger@google.com# system-level imports 1583aaf88b999306b2361ee01b0c6a7c345f2242a2commit-bot@chromium.orgimport io 169d33154b1362d89454297c21663b2403f1a75de5epoger@google.comimport json 17d23e683c51abaac6f343b0910db97caf91415bdescroggo@google.comimport os 1816f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.orgimport posixpath 1916f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.orgimport re 209d33154b1362d89454297c21663b2403f1a75de5epoger@google.com 219d33154b1362d89454297c21663b2403f1a75de5epoger@google.com 22a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.com# Key strings used in GM results JSON files (both expected-results.json and 23a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.com# actual-results.json). 24a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.com# 2516f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org# NOTE: These constants must be kept in sync with the kJsonKey_ constants in 267418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.org# gm_expectations.cpp and tools/PictureRenderer.cpp ! 277418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.org# Eric suggests: create gm/gm_expectations_constants.h containing ONLY variable 287418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.org# declarations so as to be readable by both gm/gm_expectations.cpp and Python. 29a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.com 3006e626d69edeadd03544c147272424b6d89d25aaepoger@google.com 319d33154b1362d89454297c21663b2403f1a75de5epoger@google.comJSONKEY_ACTUALRESULTS = 'actual-results' 3206e626d69edeadd03544c147272424b6d89d25aaepoger@google.com 33a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.com# Tests whose results failed to match expectations. 349d33154b1362d89454297c21663b2403f1a75de5epoger@google.comJSONKEY_ACTUALRESULTS_FAILED = 'failed' 3506e626d69edeadd03544c147272424b6d89d25aaepoger@google.com 36a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.com# Tests whose results failed to match expectations, but IGNOREFAILURE causes 37a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.com# us to take them less seriously. 389d33154b1362d89454297c21663b2403f1a75de5epoger@google.comJSONKEY_ACTUALRESULTS_FAILUREIGNORED = 'failure-ignored' 3906e626d69edeadd03544c147272424b6d89d25aaepoger@google.com 40a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.com# Tests for which we do not have any expectations. They may be new tests that 41a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.com# we haven't had a chance to check in expectations for yet, or we may have 42a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.com# consciously decided to leave them without expectations because we are unhappy 43a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.com# with the results (although we should try to move away from that, and instead 44a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.com# check in expectations with the IGNOREFAILURE flag set). 459d33154b1362d89454297c21663b2403f1a75de5epoger@google.comJSONKEY_ACTUALRESULTS_NOCOMPARISON = 'no-comparison' 4606e626d69edeadd03544c147272424b6d89d25aaepoger@google.com 47a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.com# Tests whose results matched their expectations. 489d33154b1362d89454297c21663b2403f1a75de5epoger@google.comJSONKEY_ACTUALRESULTS_SUCCEEDED = 'succeeded' 499d33154b1362d89454297c21663b2403f1a75de5epoger@google.com 5006e626d69edeadd03544c147272424b6d89d25aaepoger@google.com 5153953b457b9a4d1e3e4318a756fd40b5cb7375b0epoger@google.comJSONKEY_EXPECTEDRESULTS = 'expected-results' 5206e626d69edeadd03544c147272424b6d89d25aaepoger@google.com 53a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.com# One or more [HashType/DigestValue] pairs representing valid results for this 54a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.com# test. Typically, there will just be one pair, but we allow for multiple 55a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.com# expectations, and the test will pass if any one of them is matched. 5653953b457b9a4d1e3e4318a756fd40b5cb7375b0epoger@google.comJSONKEY_EXPECTEDRESULTS_ALLOWEDDIGESTS = 'allowed-digests' 5706e626d69edeadd03544c147272424b6d89d25aaepoger@google.com 5806e626d69edeadd03544c147272424b6d89d25aaepoger@google.com# Optional: one or more integers listing Skia bugs (under 5906e626d69edeadd03544c147272424b6d89d25aaepoger@google.com# https://code.google.com/p/skia/issues/list ) that pertain to this expectation. 6006e626d69edeadd03544c147272424b6d89d25aaepoger@google.comJSONKEY_EXPECTEDRESULTS_BUGS = 'bugs' 6106e626d69edeadd03544c147272424b6d89d25aaepoger@google.com 62a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.com# If IGNOREFAILURE is set to True, a failure of this test will be reported 63a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.com# within the FAILUREIGNORED section (thus NOT causing the buildbots to go red) 64a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.com# rather than the FAILED section (which WOULD cause the buildbots to go red). 6553953b457b9a4d1e3e4318a756fd40b5cb7375b0epoger@google.comJSONKEY_EXPECTEDRESULTS_IGNOREFAILURE = 'ignore-failure' 6653953b457b9a4d1e3e4318a756fd40b5cb7375b0epoger@google.com 6706e626d69edeadd03544c147272424b6d89d25aaepoger@google.com# Optional: a free-form text string with human-readable information about 6806e626d69edeadd03544c147272424b6d89d25aaepoger@google.com# this expectation. 6906e626d69edeadd03544c147272424b6d89d25aaepoger@google.comJSONKEY_EXPECTEDRESULTS_NOTES = 'notes' 7006e626d69edeadd03544c147272424b6d89d25aaepoger@google.com 7106e626d69edeadd03544c147272424b6d89d25aaepoger@google.com# Optional: boolean indicating whether this expectation was reviewed/approved 7206e626d69edeadd03544c147272424b6d89d25aaepoger@google.com# by a human being. 7306e626d69edeadd03544c147272424b6d89d25aaepoger@google.com# If True: a human looked at this image and approved it. 7406e626d69edeadd03544c147272424b6d89d25aaepoger@google.com# If False: this expectation was committed blind. (In such a case, please 7506e626d69edeadd03544c147272424b6d89d25aaepoger@google.com# add notes indicating why!) 7606e626d69edeadd03544c147272424b6d89d25aaepoger@google.com# If absent: this expectation was committed by a tool that didn't enforce human 7706e626d69edeadd03544c147272424b6d89d25aaepoger@google.com# review of expectations. 7806e626d69edeadd03544c147272424b6d89d25aaepoger@google.comJSONKEY_EXPECTEDRESULTS_REVIEWED = 'reviewed-by-human' 7906e626d69edeadd03544c147272424b6d89d25aaepoger@google.com 80a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.com# Allowed hash types for test expectations. 8153953b457b9a4d1e3e4318a756fd40b5cb7375b0epoger@google.comJSONKEY_HASHTYPE_BITMAP_64BITMD5 = 'bitmap-64bitMD5' 8253953b457b9a4d1e3e4318a756fd40b5cb7375b0epoger@google.com 837418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.orgJSONKEY_HEADER = 'header' 847418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.orgJSONKEY_HEADER_TYPE = 'type' 857418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.orgJSONKEY_HEADER_REVISION = 'revision' 867418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.orgJSONKEY_IMAGE_CHECKSUMALGORITHM = 'checksumAlgorithm' 877418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.orgJSONKEY_IMAGE_CHECKSUMVALUE = 'checksumValue' 887418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.orgJSONKEY_IMAGE_COMPARISONRESULT = 'comparisonResult' 897418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.orgJSONKEY_IMAGE_FILEPATH = 'filepath' 907418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.orgJSONKEY_SOURCE_TILEDIMAGES = 'tiled-images' 917418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.orgJSONKEY_SOURCE_WHOLEIMAGE = 'whole-image' 927418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.org 937418bd8cad3576294b48dd8e5015301e184c6af1commit-bot@chromium.org 9461822a25d33c18c0bd0397774451fc6ada58c310epoger@google.com# Root directory where the buildbots store their actually-generated images... 9561822a25d33c18c0bd0397774451fc6ada58c310epoger@google.com# as a publicly readable HTTP URL: 9661822a25d33c18c0bd0397774451fc6ada58c310epoger@google.comGM_ACTUALS_ROOT_HTTP_URL = ( 9761822a25d33c18c0bd0397774451fc6ada58c310epoger@google.com 'http://chromium-skia-gm.commondatastorage.googleapis.com/gm') 9861822a25d33c18c0bd0397774451fc6ada58c310epoger@google.com# as a GS URL that allows credential-protected write access: 9961822a25d33c18c0bd0397774451fc6ada58c310epoger@google.comGM_ACTUALS_ROOT_GS_URL = 'gs://chromium-skia-gm/gm' 10061822a25d33c18c0bd0397774451fc6ada58c310epoger@google.com 101d23e683c51abaac6f343b0910db97caf91415bdescroggo@google.com# Root directory where buildbots store skimage actual results json files. 102d23e683c51abaac6f343b0910db97caf91415bdescroggo@google.comSKIMAGE_ACTUALS_BASE_URL = ( 103d23e683c51abaac6f343b0910db97caf91415bdescroggo@google.com 'http://chromium-skia-gm.commondatastorage.googleapis.com/skimage/actuals') 104d23e683c51abaac6f343b0910db97caf91415bdescroggo@google.com# Root directory inside trunk where skimage expectations are stored. 105d23e683c51abaac6f343b0910db97caf91415bdescroggo@google.comSKIMAGE_EXPECTATIONS_ROOT = os.path.join('expectations', 'skimage') 106d23e683c51abaac6f343b0910db97caf91415bdescroggo@google.com 10761822a25d33c18c0bd0397774451fc6ada58c310epoger@google.com# Pattern used to assemble each image's filename 10869ac6defc1548b6b43e926bdcab9c13b333d4b16commit-bot@chromium.orgIMAGE_FILENAME_PATTERN = '(.+)_(.+)\.png' # matches (testname, config) 10961822a25d33c18c0bd0397774451fc6ada58c310epoger@google.com 11016f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org# Pattern used to create image URLs, relative to some base URL. 11116f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.orgGM_RELATIVE_URL_FORMATTER = '%s/%s/%s.png' # pass in (hash_type, test_name, 11216f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org # hash_digest) 11316f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.orgGM_RELATIVE_URL_PATTERN = '(.+)/(.+)/(.+).png' # matches (hash_type, test_name, 11416f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org # hash_digest) 11516f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.orgGM_RELATIVE_URL_RE = re.compile(GM_RELATIVE_URL_PATTERN) 11616f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org 11716f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org 11861822a25d33c18c0bd0397774451fc6ada58c310epoger@google.comdef CreateGmActualUrl(test_name, hash_type, hash_digest, 11961822a25d33c18c0bd0397774451fc6ada58c310epoger@google.com gm_actuals_root_url=GM_ACTUALS_ROOT_HTTP_URL): 12061822a25d33c18c0bd0397774451fc6ada58c310epoger@google.com """Return the URL we can use to download a particular version of 12161822a25d33c18c0bd0397774451fc6ada58c310epoger@google.com the actually-generated image for this particular GM test. 12261822a25d33c18c0bd0397774451fc6ada58c310epoger@google.com 12361822a25d33c18c0bd0397774451fc6ada58c310epoger@google.com test_name: name of the test, e.g. 'perlinnoise' 12461822a25d33c18c0bd0397774451fc6ada58c310epoger@google.com hash_type: string indicating the hash type used to generate hash_digest, 12561822a25d33c18c0bd0397774451fc6ada58c310epoger@google.com e.g. JSONKEY_HASHTYPE_BITMAP_64BITMD5 12661822a25d33c18c0bd0397774451fc6ada58c310epoger@google.com hash_digest: the hash digest of the image to retrieve 12761822a25d33c18c0bd0397774451fc6ada58c310epoger@google.com gm_actuals_root_url: root url where actual images are stored 12861822a25d33c18c0bd0397774451fc6ada58c310epoger@google.com """ 12916f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org return posixpath.join( 13016f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org gm_actuals_root_url, CreateGmRelativeUrl( 13116f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org test_name=test_name, hash_type=hash_type, hash_digest=hash_digest)) 13216f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org 13316f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org 13416f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.orgdef CreateGmRelativeUrl(test_name, hash_type, hash_digest): 13516f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org """Returns a relative URL pointing at a test result's image. 13616f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org 13716f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org Returns the URL we can use to download a particular version of 13816f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org the actually-generated image for this particular GM test, 13916f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org relative to the URL root. 14016f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org 14116f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org Args: 14216f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org test_name: name of the test, e.g. 'perlinnoise' 14316f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org hash_type: string indicating the hash type used to generate hash_digest, 14416f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org e.g. JSONKEY_HASHTYPE_BITMAP_64BITMD5 14516f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org hash_digest: the hash digest of the image to retrieve 14616f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org """ 14716f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org return GM_RELATIVE_URL_FORMATTER % (hash_type, test_name, hash_digest) 14816f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org 14916f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org 15016f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.orgdef SplitGmRelativeUrl(url): 15116f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org """Splits the relative URL into (test_name, hash_type, hash_digest) tuple. 15216f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org 15316f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org This is the inverse of CreateGmRelativeUrl(). 15416f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org 15516f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org Args: 15616f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org url: a URL generated with CreateGmRelativeUrl(). 15716f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org 15816f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org Returns: (test_name, hash_type, hash_digest) tuple. 15916f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org """ 16016f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org hash_type, test_name, hash_digest = GM_RELATIVE_URL_RE.match(url).groups() 16116f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org return (test_name, hash_type, hash_digest) 16216f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org 16361822a25d33c18c0bd0397774451fc6ada58c310epoger@google.com 1644075fd485101eea80cc67c396e6839e555ab948aepoger@google.comdef LoadFromString(file_contents): 1659d33154b1362d89454297c21663b2403f1a75de5epoger@google.com """Loads the JSON summary written out by the GM tool. 1669d33154b1362d89454297c21663b2403f1a75de5epoger@google.com Returns a dictionary keyed by the values listed as JSONKEY_ constants 1679d33154b1362d89454297c21663b2403f1a75de5epoger@google.com above.""" 1684075fd485101eea80cc67c396e6839e555ab948aepoger@google.com # TODO(epoger): we should add a version number to the JSON file to ensure 1699d33154b1362d89454297c21663b2403f1a75de5epoger@google.com # that the writer and reader agree on the schema (raising an exception 1709d33154b1362d89454297c21663b2403f1a75de5epoger@google.com # otherwise). 1714075fd485101eea80cc67c396e6839e555ab948aepoger@google.com json_dict = json.loads(file_contents) 1729d33154b1362d89454297c21663b2403f1a75de5epoger@google.com return json_dict 1734075fd485101eea80cc67c396e6839e555ab948aepoger@google.com 17416f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org 1754075fd485101eea80cc67c396e6839e555ab948aepoger@google.comdef LoadFromFile(file_path): 1764075fd485101eea80cc67c396e6839e555ab948aepoger@google.com """Loads the JSON summary written out by the GM tool. 1774075fd485101eea80cc67c396e6839e555ab948aepoger@google.com Returns a dictionary keyed by the values listed as JSONKEY_ constants 1784075fd485101eea80cc67c396e6839e555ab948aepoger@google.com above.""" 1794075fd485101eea80cc67c396e6839e555ab948aepoger@google.com file_contents = open(file_path, 'r').read() 1804075fd485101eea80cc67c396e6839e555ab948aepoger@google.com return LoadFromString(file_contents) 181a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.com 18216f418080ff6751e15e0193263149412de9c848acommit-bot@chromium.org 183a783f2bee894e09a9350a93d83b25ebe03ecca87epoger@google.comdef WriteToFile(json_dict, file_path): 18483aaf88b999306b2361ee01b0c6a7c345f2242a2commit-bot@chromium.org """Writes the JSON summary in json_dict out to file_path. 18583aaf88b999306b2361ee01b0c6a7c345f2242a2commit-bot@chromium.org 18683aaf88b999306b2361ee01b0c6a7c345f2242a2commit-bot@chromium.org The file is written Unix-style (each line ends with just LF, not CRLF); 18783aaf88b999306b2361ee01b0c6a7c345f2242a2commit-bot@chromium.org see https://code.google.com/p/skia/issues/detail?id=1815 for reasons.""" 18883aaf88b999306b2361ee01b0c6a7c345f2242a2commit-bot@chromium.org with io.open(file_path, mode='w', newline='', encoding='utf-8') as outfile: 18983aaf88b999306b2361ee01b0c6a7c345f2242a2commit-bot@chromium.org outfile.write(unicode(json.dumps(json_dict, outfile, sort_keys=True, 19083aaf88b999306b2361ee01b0c6a7c345f2242a2commit-bot@chromium.org indent=2))) 191