1# Copyright 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Base classes for a test and validator which upload results
6(reference images, error images) to cloud storage."""
7
8import os
9import re
10import tempfile
11
12from telemetry import benchmark
13from telemetry.core import bitmap
14from telemetry.page import page_test
15from telemetry.util import cloud_storage
16
17
18test_data_dir = os.path.abspath(os.path.join(
19    os.path.dirname(__file__), '..', '..', 'data', 'gpu'))
20
21default_generated_data_dir = os.path.join(test_data_dir, 'generated')
22
23error_image_cloud_storage_bucket = 'chromium-browser-gpu-tests'
24
25def _CompareScreenshotSamples(screenshot, expectations, device_pixel_ratio):
26  for expectation in expectations:
27    location = expectation["location"]
28    x = int(location[0] * device_pixel_ratio)
29    y = int(location[1] * device_pixel_ratio)
30
31    if x < 0 or y < 0 or x > screenshot.width or y > screenshot.height:
32      raise page_test.Failure(
33          'Expected pixel location [%d, %d] is out of range on [%d, %d] image' %
34          (x, y, screenshot.width, screenshot.height))
35
36    actual_color = screenshot.GetPixelColor(x, y)
37    expected_color = bitmap.RgbaColor(
38        expectation["color"][0],
39        expectation["color"][1],
40        expectation["color"][2])
41    if not actual_color.IsEqual(expected_color, expectation["tolerance"]):
42      raise page_test.Failure('Expected pixel at ' + str(location) +
43          ' to be ' +
44          str(expectation["color"]) + " but got [" +
45          str(actual_color.r) + ", " +
46          str(actual_color.g) + ", " +
47          str(actual_color.b) + "]")
48
49class ValidatorBase(page_test.PageTest):
50  def __init__(self):
51    super(ValidatorBase, self).__init__()
52    # Parameters for cloud storage reference images.
53    self.vendor_id = None
54    self.device_id = None
55    self.vendor_string = None
56    self.device_string = None
57    self.msaa = False
58
59  ###
60  ### Routines working with the local disk (only used for local
61  ### testing without a cloud storage account -- the bots do not use
62  ### this code path).
63  ###
64
65  def _UrlToImageName(self, url):
66    image_name = re.sub(r'^(http|https|file)://(/*)', '', url)
67    image_name = re.sub(r'\.\./', '', image_name)
68    image_name = re.sub(r'(\.|/|-)', '_', image_name)
69    return image_name
70
71  def _WriteImage(self, image_path, png_image):
72    output_dir = os.path.dirname(image_path)
73    if not os.path.exists(output_dir):
74      os.makedirs(output_dir)
75    png_image.WritePngFile(image_path)
76
77  def _WriteErrorImages(self, img_dir, img_name, screenshot, ref_png):
78    full_image_name = img_name + '_' + str(self.options.build_revision)
79    full_image_name = full_image_name + '.png'
80
81    # Always write the failing image.
82    self._WriteImage(
83        os.path.join(img_dir, 'FAIL_' + full_image_name), screenshot)
84
85    if ref_png:
86      # Save the reference image.
87      # This ensures that we get the right revision number.
88      self._WriteImage(
89          os.path.join(img_dir, full_image_name), ref_png)
90
91      # Save the difference image.
92      diff_png = screenshot.Diff(ref_png)
93      self._WriteImage(
94          os.path.join(img_dir, 'DIFF_' + full_image_name), diff_png)
95
96  ###
97  ### Cloud storage code path -- the bots use this.
98  ###
99
100  def _ComputeGpuInfo(self, tab):
101    if ((self.vendor_id and self.device_id) or
102        (self.vendor_string and self.device_string)):
103      return
104    browser = tab.browser
105    if not browser.supports_system_info:
106      raise Exception('System info must be supported by the browser')
107    system_info = browser.GetSystemInfo()
108    if not system_info.gpu:
109      raise Exception('GPU information was absent')
110    device = system_info.gpu.devices[0]
111    if device.vendor_id and device.device_id:
112      self.vendor_id = device.vendor_id
113      self.device_id = device.device_id
114    elif device.vendor_string and device.device_string:
115      self.vendor_string = device.vendor_string
116      self.device_string = device.device_string
117    else:
118      raise Exception('GPU device information was incomplete')
119    self.msaa = not (
120        'disable_multisampling' in system_info.gpu.driver_bug_workarounds)
121
122  def _FormatGpuInfo(self, tab):
123    self._ComputeGpuInfo(tab)
124    msaa_string = '_msaa' if self.msaa else '_non_msaa'
125    if self.vendor_id:
126      return '%s_%04x_%04x%s' % (
127        self.options.os_type, self.vendor_id, self.device_id, msaa_string)
128    else:
129      return '%s_%s_%s%s' % (
130        self.options.os_type, self.vendor_string, self.device_string,
131        msaa_string)
132
133  def _FormatReferenceImageName(self, img_name, page, tab):
134    return '%s_v%s_%s.png' % (
135      img_name,
136      page.revision,
137      self._FormatGpuInfo(tab))
138
139  def _UploadBitmapToCloudStorage(self, bucket, name, bitmap, public=False):
140    # This sequence of steps works on all platforms to write a temporary
141    # PNG to disk, following the pattern in bitmap_unittest.py. The key to
142    # avoiding PermissionErrors seems to be to not actually try to write to
143    # the temporary file object, but to re-open its name for all operations.
144    temp_file = tempfile.NamedTemporaryFile().name
145    bitmap.WritePngFile(temp_file)
146    cloud_storage.Insert(bucket, name, temp_file, publicly_readable=public)
147
148  def _ConditionallyUploadToCloudStorage(self, img_name, page, tab, screenshot):
149    """Uploads the screenshot to cloud storage as the reference image
150    for this test, unless it already exists. Returns True if the
151    upload was actually performed."""
152    if not self.options.refimg_cloud_storage_bucket:
153      raise Exception('--refimg-cloud-storage-bucket argument is required')
154    cloud_name = self._FormatReferenceImageName(img_name, page, tab)
155    if not cloud_storage.Exists(self.options.refimg_cloud_storage_bucket,
156                                cloud_name):
157      self._UploadBitmapToCloudStorage(self.options.refimg_cloud_storage_bucket,
158                                       cloud_name,
159                                       screenshot)
160      return True
161    return False
162
163  def _DownloadFromCloudStorage(self, img_name, page, tab):
164    """Downloads the reference image for the given test from cloud
165    storage, returning it as a Telemetry Bitmap object."""
166    # TODO(kbr): there's a race condition between the deletion of the
167    # temporary file and gsutil's overwriting it.
168    if not self.options.refimg_cloud_storage_bucket:
169      raise Exception('--refimg-cloud-storage-bucket argument is required')
170    temp_file = tempfile.NamedTemporaryFile().name
171    cloud_storage.Get(self.options.refimg_cloud_storage_bucket,
172                      self._FormatReferenceImageName(img_name, page, tab),
173                      temp_file)
174    return bitmap.Bitmap.FromPngFile(temp_file)
175
176  def _UploadErrorImagesToCloudStorage(self, image_name, screenshot, ref_img):
177    """For a failing run, uploads the failing image, reference image (if
178    supplied), and diff image (if reference image was supplied) to cloud
179    storage. This subsumes the functionality of the
180    archive_gpu_pixel_test_results.py script."""
181    machine_name = re.sub('\W+', '_', self.options.test_machine_name)
182    upload_dir = '%s_%s_telemetry' % (self.options.build_revision, machine_name)
183    base_bucket = '%s/runs/%s' % (error_image_cloud_storage_bucket, upload_dir)
184    image_name_with_revision = '%s_%s.png' % (
185      image_name, self.options.build_revision)
186    self._UploadBitmapToCloudStorage(
187      base_bucket + '/gen', image_name_with_revision, screenshot,
188      public=True)
189    if ref_img:
190      self._UploadBitmapToCloudStorage(
191        base_bucket + '/ref', image_name_with_revision, ref_img, public=True)
192      diff_img = screenshot.Diff(ref_img)
193      self._UploadBitmapToCloudStorage(
194        base_bucket + '/diff', image_name_with_revision, diff_img,
195        public=True)
196    print ('See http://%s.commondatastorage.googleapis.com/'
197           'view_test_results.html?%s for this run\'s test results') % (
198      error_image_cloud_storage_bucket, upload_dir)
199
200  def _ValidateScreenshotSamples(self, url,
201                                 screenshot, expectations, device_pixel_ratio):
202    """Samples the given screenshot and verifies pixel color values.
203       The sample locations and expected color values are given in expectations.
204       In case any of the samples do not match the expected color, it raises
205       a Failure and dumps the screenshot locally or cloud storage depending on
206       what machine the test is being run."""
207    try:
208      _CompareScreenshotSamples(screenshot, expectations, device_pixel_ratio)
209    except page_test.Failure:
210      image_name = self._UrlToImageName(url)
211      if self.options.test_machine_name:
212        self._UploadErrorImagesToCloudStorage(image_name, screenshot, None)
213      else:
214        self._WriteErrorImages(self.options.generated_dir, image_name,
215                               screenshot, None)
216      raise
217
218
219class TestBase(benchmark.Benchmark):
220  @classmethod
221  def AddTestCommandLineArgs(cls, group):
222    group.add_option('--build-revision',
223        help='Chrome revision being tested.',
224        default="unknownrev")
225    group.add_option('--upload-refimg-to-cloud-storage',
226        dest='upload_refimg_to_cloud_storage',
227        action='store_true', default=False,
228        help='Upload resulting images to cloud storage as reference images')
229    group.add_option('--download-refimg-from-cloud-storage',
230        dest='download_refimg_from_cloud_storage',
231        action='store_true', default=False,
232        help='Download reference images from cloud storage')
233    group.add_option('--refimg-cloud-storage-bucket',
234        help='Name of the cloud storage bucket to use for reference images; '
235        'required with --upload-refimg-to-cloud-storage and '
236        '--download-refimg-from-cloud-storage. Example: '
237        '"chromium-gpu-archive/reference-images"')
238    group.add_option('--os-type',
239        help='Type of operating system on which the pixel test is being run, '
240        'used only to distinguish different operating systems with the same '
241        'graphics card. Any value is acceptable, but canonical values are '
242        '"win", "mac", and "linux", and probably, eventually, "chromeos" '
243        'and "android").',
244        default='')
245    group.add_option('--test-machine-name',
246        help='Name of the test machine. Specifying this argument causes this '
247        'script to upload failure images and diffs to cloud storage directly, '
248        'instead of relying on the archive_gpu_pixel_test_results.py script.',
249        default='')
250    group.add_option('--generated-dir',
251        help='Overrides the default on-disk location for generated test images '
252        '(only used for local testing without a cloud storage account)',
253        default=default_generated_data_dir)
254