1# Copyright 2014 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import os 6import unittest 7 8from telemetry import benchmark 9from telemetry.core import platform 10from telemetry.core import wpr_modes 11from telemetry.page import page as page_module 12from telemetry.page import page_set 13from telemetry.results import page_test_results 14from telemetry.timeline import model as model_module 15from telemetry.timeline import async_slice 16from telemetry.unittest import options_for_unittests 17from telemetry.unittest import page_test_test_case 18from telemetry.value import scalar 19from telemetry.web_perf import timeline_based_measurement as tbm_module 20from telemetry.web_perf import timeline_interaction_record as tir_module 21from telemetry.web_perf.metrics import timeline_based_metric 22 23 24class FakeFastMetric(timeline_based_metric.TimelineBasedMetric): 25 26 def AddResults(self, model, renderer_thread, interaction_records, results): 27 results.AddValue(scalar.ScalarValue( 28 results.current_page, 'FakeFastMetric', 'ms', 1)) 29 results.AddValue(scalar.ScalarValue( 30 results.current_page, 'FastMetricRecords', 'count', 31 len(interaction_records))) 32 33 34class FakeSmoothMetric(timeline_based_metric.TimelineBasedMetric): 35 36 def AddResults(self, model, renderer_thread, interaction_records, results): 37 results.AddValue(scalar.ScalarValue( 38 results.current_page, 'FakeSmoothMetric', 'ms', 1)) 39 results.AddValue(scalar.ScalarValue( 40 results.current_page, 'SmoothMetricRecords', 'count', 41 len(interaction_records))) 42 43 44class FakeLoadingMetric(timeline_based_metric.TimelineBasedMetric): 45 46 def AddResults(self, model, renderer_thread, interaction_records, results): 47 results.AddValue(scalar.ScalarValue( 48 results.current_page, 'FakeLoadingMetric', 'ms', 2)) 49 results.AddValue(scalar.ScalarValue( 50 results.current_page, 'LoadingMetricRecords', 'count', 51 len(interaction_records))) 52 53 54def GetMetricFromMetricType(metric_type): 55 if metric_type == tir_module.IS_FAST: 56 return FakeFastMetric() 57 if metric_type == tir_module.IS_SMOOTH: 58 return FakeSmoothMetric() 59 if metric_type == tir_module.IS_RESPONSIVE: 60 return FakeLoadingMetric() 61 raise Exception('Unrecognized metric type: %s' % metric_type) 62 63 64class TimelineBasedMetricTestData(object): 65 66 def __init__(self): 67 self._model = model_module.TimelineModel() 68 renderer_process = self._model.GetOrCreateProcess(1) 69 self._renderer_thread = renderer_process.GetOrCreateThread(2) 70 self._renderer_thread.name = 'CrRendererMain' 71 self._results = page_test_results.PageTestResults() 72 self._metric = None 73 self._ps = None 74 75 @property 76 def results(self): 77 return self._results 78 79 @property 80 def metric(self): 81 return self._metric 82 83 def AddInteraction(self, marker='', ts=0, duration=5): 84 self._renderer_thread.async_slices.append(async_slice.AsyncSlice( 85 'category', marker, timestamp=ts, duration=duration, 86 start_thread=self._renderer_thread, end_thread=self._renderer_thread, 87 thread_start=ts, thread_duration=duration)) 88 89 def FinalizeImport(self): 90 self._model.FinalizeImport() 91 self._metric = tbm_module._TimelineBasedMetrics( # pylint: disable=W0212 92 self._model, self._renderer_thread, GetMetricFromMetricType) 93 self._ps = page_set.PageSet(file_path=os.path.dirname(__file__)) 94 self._ps.AddPageWithDefaultRunNavigate('http://www.bar.com/') 95 self._results.WillRunPage(self._ps.pages[0]) 96 97 def AddResults(self): 98 self._metric.AddResults(self._results) 99 self._results.DidRunPage(self._ps.pages[0]) 100 101 102class TimelineBasedMetricsTests(unittest.TestCase): 103 104 def testFindTimelineInteractionRecords(self): 105 d = TimelineBasedMetricTestData() 106 d.AddInteraction(ts=0, duration=20, 107 marker='Interaction.LogicalName1/is_smooth') 108 d.AddInteraction(ts=25, duration=5, 109 marker='Interaction.LogicalName2/is_responsive') 110 d.AddInteraction(ts=50, duration=15, 111 marker='Interaction.LogicalName3/is_fast') 112 d.FinalizeImport() 113 interactions = d.metric.FindTimelineInteractionRecords() 114 self.assertEquals(3, len(interactions)) 115 self.assertTrue(interactions[0].is_smooth) 116 self.assertEquals(0, interactions[0].start) 117 self.assertEquals(20, interactions[0].end) 118 119 self.assertTrue(interactions[1].is_responsive) 120 self.assertEquals(25, interactions[1].start) 121 self.assertEquals(30, interactions[1].end) 122 123 self.assertTrue(interactions[2].is_fast) 124 self.assertEquals(50, interactions[2].start) 125 self.assertEquals(65, interactions[2].end) 126 127 def testAddResults(self): 128 d = TimelineBasedMetricTestData() 129 d.AddInteraction(ts=0, duration=20, 130 marker='Interaction.LogicalName1/is_smooth') 131 d.AddInteraction(ts=25, duration=5, 132 marker='Interaction.LogicalName2/is_responsive') 133 d.AddInteraction(ts=50, duration=15, 134 marker='Interaction.LogicalName3/is_fast') 135 d.FinalizeImport() 136 d.AddResults() 137 self.assertEquals(1, len(d.results.FindAllPageSpecificValuesNamed( 138 'LogicalName1-FakeSmoothMetric'))) 139 self.assertEquals(1, len(d.results.FindAllPageSpecificValuesNamed( 140 'LogicalName2-FakeLoadingMetric'))) 141 self.assertEquals(1, len(d.results.FindAllPageSpecificValuesNamed( 142 'LogicalName3-FakeFastMetric'))) 143 144 def testNoInteractions(self): 145 d = TimelineBasedMetricTestData() 146 d.FinalizeImport() 147 self.assertRaises(tbm_module.InvalidInteractions, d.AddResults) 148 149 def testDuplicateUnrepeatableInteractions(self): 150 d = TimelineBasedMetricTestData() 151 d.AddInteraction(ts=10, duration=5, 152 marker='Interaction.LogicalName/is_smooth') 153 d.AddInteraction(ts=20, duration=5, 154 marker='Interaction.LogicalName/is_smooth') 155 d.FinalizeImport() 156 self.assertRaises(tbm_module.InvalidInteractions, d.AddResults) 157 158 def testDuplicateRepeatableInteractions(self): 159 d = TimelineBasedMetricTestData() 160 d.AddInteraction(ts=10, duration=5, 161 marker='Interaction.LogicalName/is_smooth,repeatable') 162 d.AddInteraction(ts=20, duration=5, 163 marker='Interaction.LogicalName/is_smooth,repeatable') 164 d.FinalizeImport() 165 d.AddResults() 166 self.assertEquals(1, len(d.results.pages_that_succeeded)) 167 168 def testDuplicateRepeatableInteractionsWithDifferentMetrics(self): 169 d = TimelineBasedMetricTestData() 170 171 responsive_marker = 'Interaction.LogicalName/is_responsive,repeatable' 172 d.AddInteraction(ts=10, duration=5, marker=responsive_marker) 173 smooth_marker = 'Interaction.LogicalName/is_smooth,repeatable' 174 d.AddInteraction(ts=20, duration=5, marker=smooth_marker) 175 d.FinalizeImport() 176 self.assertRaises(tbm_module.InvalidInteractions, d.AddResults) 177 178 179class TestTimelinebasedMeasurementPage(page_module.Page): 180 181 def __init__(self, ps, base_dir, trigger_animation=False, 182 trigger_jank=False, trigger_slow=False): 183 super(TestTimelinebasedMeasurementPage, self).__init__( 184 'file://interaction_enabled_page.html', ps, base_dir) 185 self._trigger_animation = trigger_animation 186 self._trigger_jank = trigger_jank 187 self._trigger_slow = trigger_slow 188 189 def RunSmoothness(self, action_runner): 190 if self._trigger_animation: 191 action_runner.TapElement('#animating-button') 192 action_runner.WaitForJavaScriptCondition('window.animationDone') 193 if self._trigger_jank: 194 action_runner.TapElement('#jank-button') 195 action_runner.WaitForJavaScriptCondition('window.jankScriptDone') 196 if self._trigger_slow: 197 action_runner.TapElement('#slow-button') 198 action_runner.WaitForJavaScriptCondition('window.slowScriptDone') 199 200 201class TimelineBasedMeasurementTest(page_test_test_case.PageTestTestCase): 202 203 def setUp(self): 204 self._options = options_for_unittests.GetCopy() 205 self._options.browser_options.wpr_mode = wpr_modes.WPR_OFF 206 207 def testSmoothnessTimelineBasedMeasurementForSmoke(self): 208 ps = self.CreateEmptyPageSet() 209 ps.AddPage(TestTimelinebasedMeasurementPage( 210 ps, ps.base_dir, trigger_animation=True)) 211 212 measurement = tbm_module.TimelineBasedMeasurement() 213 results = self.RunMeasurement(measurement, ps, 214 options=self._options) 215 216 self.assertEquals(0, len(results.failures)) 217 v = results.FindAllPageSpecificValuesNamed('CenterAnimation-jank') 218 self.assertEquals(len(v), 1) 219 v = results.FindAllPageSpecificValuesNamed('DrawerAnimation-jank') 220 self.assertEquals(len(v), 1) 221 222 def testFastTimelineBasedMeasurementForSmoke(self): 223 ps = self.CreateEmptyPageSet() 224 ps.AddPage(TestTimelinebasedMeasurementPage( 225 ps, ps.base_dir, trigger_slow=True)) 226 227 measurement = tbm_module.TimelineBasedMeasurement() 228 results = self.RunMeasurement(measurement, ps, options=self._options) 229 230 self.assertEquals([], results.failures) 231 expected_names = set([ 232 'SlowThreadJsRun-fast-duration', 233 'SlowThreadJsRun-fast-idle_time', 234 'SlowThreadJsRun-fast-incremental_marking', 235 'SlowThreadJsRun-fast-incremental_marking_outside_idle', 236 'SlowThreadJsRun-fast-mark_compactor', 237 'SlowThreadJsRun-fast-mark_compactor_outside_idle', 238 'SlowThreadJsRun-fast-scavenger', 239 'SlowThreadJsRun-fast-scavenger_outside_idle', 240 'SlowThreadJsRun-fast-total_garbage_collection', 241 'SlowThreadJsRun-fast-total_garbage_collection_outside_idle', 242 ]) 243 if platform.GetHostPlatform().GetOSName() != 'win': 244 # CPU metric is only supported non-Windows platforms. 245 expected_names.add('SlowThreadJsRun-fast-cpu_time') 246 self.assertEquals( 247 expected_names, set(v.name for v in results.all_page_specific_values)) 248 249 # In interaction_enabled_page.html, the "slow" interaction executes 250 # a loop with window.performance.now() to wait 200ms. 251 # fast-duration measures wall time so its value should be at least 200ms. 252 v = results.FindAllPageSpecificValuesNamed('SlowThreadJsRun-fast-duration') 253 self.assertGreaterEqual(v[0].value, 200.0) 254 255 # Disabled since mainthread_jank metric is not supported on windows platform. 256 @benchmark.Disabled('win') 257 def testMainthreadJankTimelineBasedMeasurement(self): 258 ps = self.CreateEmptyPageSet() 259 ps.AddPage(TestTimelinebasedMeasurementPage( 260 ps, ps.base_dir, trigger_jank=True)) 261 262 measurement = tbm_module.TimelineBasedMeasurement() 263 results = self.RunMeasurement(measurement, ps, 264 options=self._options) 265 self.assertEquals(0, len(results.failures)) 266 267 # In interaction_enabled_page.html, we create a jank loop based on 268 # window.performance.now() (basically loop for x milliseconds). 269 # Since window.performance.now() uses wall-time instead of thread time, 270 # we only assert the biggest jank > 50ms here to account for the fact 271 # that the browser may deschedule during the jank loop. 272 v = results.FindAllPageSpecificValuesNamed( 273 'JankThreadJSRun-responsive-biggest_jank_thread_time') 274 self.assertGreaterEqual(v[0].value, 50) 275 276 v = results.FindAllPageSpecificValuesNamed( 277 'JankThreadJSRun-responsive-total_big_jank_thread_time') 278 self.assertGreaterEqual(v[0].value, 50) 279