1# Copyright 2014 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import subprocess 6 7from telemetry.core import bitmap 8from telemetry.core import platform 9from telemetry.util import cloud_storage 10 11HIGHLIGHT_ORANGE_FRAME = bitmap.WEB_PAGE_TEST_ORANGE 12 13class BoundingBoxNotFoundException(Exception): 14 pass 15 16 17class Video(object): 18 """Utilities for storing and interacting with the video capture.""" 19 20 def __init__(self, video_file_obj): 21 assert video_file_obj.delete 22 assert not video_file_obj.close_called 23 self._video_file_obj = video_file_obj 24 self._tab_contents_bounding_box = None 25 26 def UploadToCloudStorage(self, bucket, target_path): 27 """Uploads video file to cloud storage. 28 29 Args: 30 target_path: Path indicating where to store the file in cloud storage. 31 """ 32 cloud_storage.Insert(bucket, target_path, self._video_file_obj.name) 33 34 def GetVideoFrameIter(self): 35 """Returns the iteration for processing the video capture. 36 37 This looks for the initial color flash in the first frame to establish the 38 tab content boundaries and then omits all frames displaying the flash. 39 40 Yields: 41 (time_ms, bitmap) tuples representing each video keyframe. Only the first 42 frame in a run of sequential duplicate bitmaps is typically included. 43 time_ms is milliseconds since navigationStart. 44 bitmap is a telemetry.core.Bitmap. 45 """ 46 frame_generator = self._FramesFromMp4(self._video_file_obj.name) 47 48 # Flip through frames until we find the initial tab contents flash. 49 content_box = None 50 for _, bmp in frame_generator: 51 content_box = self._FindHighlightBoundingBox( 52 bmp, HIGHLIGHT_ORANGE_FRAME) 53 if content_box: 54 break 55 56 if not content_box: 57 raise BoundingBoxNotFoundException( 58 'Failed to identify tab contents in video capture.') 59 60 # Flip through frames until the flash goes away and emit that as frame 0. 61 timestamp = 0 62 for timestamp, bmp in frame_generator: 63 if not self._FindHighlightBoundingBox(bmp, HIGHLIGHT_ORANGE_FRAME): 64 yield 0, bmp.Crop(*content_box) 65 break 66 67 start_time = timestamp 68 for timestamp, bmp in frame_generator: 69 yield timestamp - start_time, bmp.Crop(*content_box) 70 71 def _FindHighlightBoundingBox(self, bmp, color, bounds_tolerance=8, 72 color_tolerance=8): 73 """Returns the bounding box of the content highlight of the given color. 74 75 Raises: 76 BoundingBoxNotFoundException if the hightlight could not be found. 77 """ 78 content_box, pixel_count = bmp.GetBoundingBox(color, 79 tolerance=color_tolerance) 80 81 if not content_box: 82 return None 83 84 # We assume arbitrarily that tabs are all larger than 200x200. If this 85 # fails it either means that assumption has changed or something is 86 # awry with our bounding box calculation. 87 if content_box[2] < 200 or content_box[3] < 200: 88 raise BoundingBoxNotFoundException('Unexpectedly small tab contents.') 89 90 # TODO(tonyg): Can this threshold be increased? 91 if pixel_count < 0.9 * content_box[2] * content_box[3]: 92 raise BoundingBoxNotFoundException( 93 'Low count of pixels in tab contents matching expected color.') 94 95 # Since we allow some fuzziness in bounding box finding, we want to make 96 # sure that the bounds are always stable across a run. So we cache the 97 # first box, whatever it may be. 98 # 99 # This relies on the assumption that since Telemetry doesn't know how to 100 # resize the window, we should always get the same content box for a tab. 101 # If this assumption changes, this caching needs to be reworked. 102 if not self._tab_contents_bounding_box: 103 self._tab_contents_bounding_box = content_box 104 105 # Verify that there is only minor variation in the bounding box. If it's 106 # just a few pixels, we can assume it's due to compression artifacts. 107 for x, y in zip(self._tab_contents_bounding_box, content_box): 108 if abs(x - y) > bounds_tolerance: 109 # If this fails, it means either that either the above assumption has 110 # changed or something is awry with our bounding box calculation. 111 raise BoundingBoxNotFoundException( 112 'Unexpected change in tab contents box.') 113 114 return self._tab_contents_bounding_box 115 116 def _FramesFromMp4(self, mp4_file): 117 host_platform = platform.GetHostPlatform() 118 if not host_platform.CanLaunchApplication('avconv'): 119 host_platform.InstallApplication('avconv') 120 121 def GetDimensions(video): 122 proc = subprocess.Popen(['avconv', '-i', video], stderr=subprocess.PIPE) 123 dimensions = None 124 output = '' 125 for line in proc.stderr.readlines(): 126 output += line 127 if 'Video:' in line: 128 dimensions = line.split(',')[2] 129 dimensions = map(int, dimensions.split()[0].split('x')) 130 break 131 proc.communicate() 132 assert dimensions, ('Failed to determine video dimensions. output=%s' % 133 output) 134 return dimensions 135 136 def GetFrameTimestampMs(stderr): 137 """Returns the frame timestamp in integer milliseconds from the dump log. 138 139 The expected line format is: 140 ' dts=1.715 pts=1.715\n' 141 142 We have to be careful to only read a single timestamp per call to avoid 143 deadlock because avconv interleaves its writes to stdout and stderr. 144 """ 145 while True: 146 line = '' 147 next_char = '' 148 while next_char != '\n': 149 next_char = stderr.read(1) 150 line += next_char 151 if 'pts=' in line: 152 return int(1000 * float(line.split('=')[-1])) 153 154 dimensions = GetDimensions(mp4_file) 155 frame_length = dimensions[0] * dimensions[1] * 3 156 frame_data = bytearray(frame_length) 157 158 # Use rawvideo so that we don't need any external library to parse frames. 159 proc = subprocess.Popen(['avconv', '-i', mp4_file, '-vcodec', 160 'rawvideo', '-pix_fmt', 'rgb24', '-dump', 161 '-loglevel', 'debug', '-f', 'rawvideo', '-'], 162 stderr=subprocess.PIPE, stdout=subprocess.PIPE) 163 while True: 164 num_read = proc.stdout.readinto(frame_data) 165 if not num_read: 166 raise StopIteration 167 assert num_read == len(frame_data), 'Unexpected frame size: %d' % num_read 168 yield (GetFrameTimestampMs(proc.stderr), 169 bitmap.Bitmap(3, dimensions[0], dimensions[1], frame_data)) 170