12a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Copyright (c) 2012 The Chromium Authors. All rights reserved. 22a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 32a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# found in the LICENSE file. 44e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)from datetime import datetime 54e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)import glob 64e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)import optparse 74e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)import os 84e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)import re 94e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 105d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)import cloud_storage_test_base 11f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles)import page_sets 12effb81e5f8246d0db0270817048dc992db66e9fbBen Murdochimport pixel_expectations 135d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 14116680a4aac90f2aa7413d9095a592090648e557Ben Murdochfrom telemetry import benchmark 15a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles)from telemetry.core import bitmap 162a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from telemetry.page import page_test 17116680a4aac90f2aa7413d9095a592090648e557Ben Murdochfrom telemetry.util import cloud_storage 18116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch 194e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 204e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)test_data_dir = os.path.abspath(os.path.join( 214e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) os.path.dirname(__file__), '..', '..', 'data', 'gpu')) 224e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 234e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)default_reference_image_dir = os.path.join(test_data_dir, 'gpu_reference') 244e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 254e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)test_harness_script = r""" 264e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) var domAutomationController = {}; 274e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 284e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) domAutomationController._succeeded = false; 294e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) domAutomationController._finished = false; 302a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 314e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) domAutomationController.setAutomationId = function(id) {} 324e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 334e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) domAutomationController.send = function(msg) { 344e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) domAutomationController._finished = true; 354e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 364e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if(msg.toLowerCase() == "success") { 374e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) domAutomationController._succeeded = true; 384e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } else { 394e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) domAutomationController._succeeded = false; 404e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 414e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) } 424e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 434e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) window.domAutomationController = domAutomationController; 444e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)""" 45eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 462a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)class PixelTestFailure(Exception): 472a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) pass 482a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 494e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)def _DidTestSucceed(tab): 504e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return tab.EvaluateJavaScript('domAutomationController._succeeded') 51eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 5223730a6e56a168d1879203e4b3819bb36e3d8f1fTorne (Richard Coles)class _PixelValidator(cloud_storage_test_base.ValidatorBase): 534e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) def CustomizeBrowserOptions(self, options): 544e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) options.AppendExtraBrowserArgs('--enable-gpu-benchmarking') 554e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 566e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) def ValidateAndMeasurePage(self, page, tab, results): 574e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if not _DidTestSucceed(tab): 584e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) raise page_test.Failure('Page indicated a failure') 594e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 604e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if not tab.screenshot_supported: 614e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) raise page_test.Failure('Browser does not support screenshot capture') 624e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 634e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) screenshot = tab.Screenshot(5) 644e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 654e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if not screenshot: 664e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) raise page_test.Failure('Could not capture screenshot') 674e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 684e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if hasattr(page, 'test_rect'): 694e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) screenshot = screenshot.Crop( 704e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) page.test_rect[0], page.test_rect[1], 714e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) page.test_rect[2], page.test_rect[3]) 724e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 735d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) image_name = self._UrlToImageName(page.display_name) 745d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 755d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if self.options.upload_refimg_to_cloud_storage: 765d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if self._ConditionallyUploadToCloudStorage(image_name, page, tab, 775d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) screenshot): 785d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # This is the new reference image; there's nothing to compare against. 795d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ref_png = screenshot 805d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) else: 815d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # There was a preexisting reference image, so we might as well 825d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # compare against it. 835d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ref_png = self._DownloadFromCloudStorage(image_name, page, tab) 845d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) elif self.options.download_refimg_from_cloud_storage: 855d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # This bot doesn't have the ability to properly generate a 865d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # reference image, so download it from cloud storage. 87cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) try: 88cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) ref_png = self._DownloadFromCloudStorage(image_name, page, tab) 89cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) except cloud_storage.NotFoundError as e: 90cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # There is no reference image yet in cloud storage. This 91cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # happens when the revision of the test is incremented or when 92cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # a new test is added, because the trybots are not allowed to 93cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # produce reference images, only the bots on the main 94cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # waterfalls. Report this as a failure so the developer has to 95cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # take action by explicitly suppressing the failure and 96cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # removing the suppression once the reference images have been 97cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # generated. Otherwise silent failures could happen for long 98cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) # periods of time. 99cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) raise page_test.Failure('Could not find image %s in cloud storage' % 100cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) image_name) 1015d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) else: 1025d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) # Legacy path using on-disk results. 1035d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) ref_png = self._GetReferenceImage(self.options.reference_dir, 1045d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) image_name, page.revision, screenshot) 1054e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1064e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) # Test new snapshot against existing reference image 1074e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if not ref_png.IsEqual(screenshot, tolerance=2): 1085d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if self.options.test_machine_name: 1095d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._UploadErrorImagesToCloudStorage(image_name, screenshot, ref_png) 1105d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) else: 1115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._WriteErrorImages(self.options.generated_dir, image_name, 1125d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) screenshot, ref_png) 1134e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) raise page_test.Failure('Reference image did not match captured screen') 1144e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1155d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def _DeleteOldReferenceImages(self, ref_image_path, cur_revision): 1164e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if not cur_revision: 1174e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return 1184e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1194e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) old_revisions = glob.glob(ref_image_path + "_*.png") 1204e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) for rev_path in old_revisions: 1214e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) m = re.match(r'^.*_(\d+)\.png$', rev_path) 1224e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if m and int(m.group(1)) < cur_revision: 1234e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) print 'Found deprecated reference image. Deleting rev ' + m.group(1) 1244e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) os.remove(rev_path) 1254e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1265d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) def _GetReferenceImage(self, img_dir, img_name, cur_revision, screenshot): 1274e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if not cur_revision: 1284e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) cur_revision = 0 1294e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1304e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) image_path = os.path.join(img_dir, img_name) 1314e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1325d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._DeleteOldReferenceImages(image_path, cur_revision) 1334e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1344e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) image_path = image_path + '_' + str(cur_revision) + '.png' 1354e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1364e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) try: 137a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) ref_png = bitmap.Bitmap.FromPngFile(image_path) 1384e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) except IOError: 1394e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) ref_png = None 1404e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1414e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if ref_png: 1424e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return ref_png 1434e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1444e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) print 'Reference image not found. Writing tab contents as reference.' 1454e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1465d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) self._WriteImage(image_path, screenshot) 1474e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return screenshot 1484e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1495d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)class Pixel(cloud_storage_test_base.TestBase): 15023730a6e56a168d1879203e4b3819bb36e3d8f1fTorne (Richard Coles) test = _PixelValidator 1514e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 152a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) @classmethod 153a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) def AddTestCommandLineArgs(cls, group): 154a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) super(Pixel, cls).AddTestCommandLineArgs(group) 1554e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) group.add_option('--reference-dir', 1565d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) help='Overrides the default on-disk location for reference images ' 1575d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '(only used for local testing without a cloud storage account)', 1584e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) default=default_reference_image_dir) 1594e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1601e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) def CreatePageSet(self, options): 1611320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci page_set = page_sets.PixelTestsPageSet() 1624e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) for page in page_set.pages: 1634e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) page.script_to_evaluate_on_commit = test_harness_script 1641e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) return page_set 165effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch 166effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch def CreateExpectations(self, page_set): 167effb81e5f8246d0db0270817048dc992db66e9fbBen Murdoch return pixel_expectations.PixelExpectations() 168