1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import os
6import unittest
7
8from telemetry import benchmark
9from telemetry.core import platform
10from telemetry.core import wpr_modes
11from telemetry.page import page as page_module
12from telemetry.page import page_set
13from telemetry.results import page_test_results
14from telemetry.timeline import model as model_module
15from telemetry.timeline import async_slice
16from telemetry.unittest import options_for_unittests
17from telemetry.unittest import page_test_test_case
18from telemetry.value import scalar
19from telemetry.web_perf import timeline_based_measurement as tbm_module
20from telemetry.web_perf import timeline_interaction_record as tir_module
21from telemetry.web_perf.metrics import timeline_based_metric
22
23
24class FakeFastMetric(timeline_based_metric.TimelineBasedMetric):
25
26  def AddResults(self, model, renderer_thread, interaction_records, results):
27    results.AddValue(scalar.ScalarValue(
28        results.current_page, 'FakeFastMetric', 'ms', 1))
29    results.AddValue(scalar.ScalarValue(
30        results.current_page, 'FastMetricRecords', 'count',
31        len(interaction_records)))
32
33
34class FakeSmoothMetric(timeline_based_metric.TimelineBasedMetric):
35
36  def AddResults(self, model, renderer_thread, interaction_records, results):
37    results.AddValue(scalar.ScalarValue(
38        results.current_page, 'FakeSmoothMetric', 'ms', 1))
39    results.AddValue(scalar.ScalarValue(
40        results.current_page, 'SmoothMetricRecords', 'count',
41        len(interaction_records)))
42
43
44class FakeLoadingMetric(timeline_based_metric.TimelineBasedMetric):
45
46  def AddResults(self, model, renderer_thread, interaction_records, results):
47    results.AddValue(scalar.ScalarValue(
48        results.current_page, 'FakeLoadingMetric', 'ms', 2))
49    results.AddValue(scalar.ScalarValue(
50        results.current_page, 'LoadingMetricRecords', 'count',
51        len(interaction_records)))
52
53
54def GetMetricFromMetricType(metric_type):
55  if metric_type == tir_module.IS_FAST:
56    return FakeFastMetric()
57  if metric_type == tir_module.IS_SMOOTH:
58    return FakeSmoothMetric()
59  if metric_type == tir_module.IS_RESPONSIVE:
60    return FakeLoadingMetric()
61  raise Exception('Unrecognized metric type: %s' % metric_type)
62
63
64class TimelineBasedMetricTestData(object):
65
66  def __init__(self):
67    self._model = model_module.TimelineModel()
68    renderer_process = self._model.GetOrCreateProcess(1)
69    self._renderer_thread = renderer_process.GetOrCreateThread(2)
70    self._renderer_thread.name = 'CrRendererMain'
71    self._results = page_test_results.PageTestResults()
72    self._metric = None
73    self._ps = None
74
75  @property
76  def results(self):
77    return self._results
78
79  @property
80  def metric(self):
81    return self._metric
82
83  def AddInteraction(self, marker='', ts=0, duration=5):
84    self._renderer_thread.async_slices.append(async_slice.AsyncSlice(
85        'category', marker, timestamp=ts, duration=duration,
86        start_thread=self._renderer_thread, end_thread=self._renderer_thread,
87        thread_start=ts, thread_duration=duration))
88
89  def FinalizeImport(self):
90    self._model.FinalizeImport()
91    self._metric = tbm_module._TimelineBasedMetrics(  # pylint: disable=W0212
92        self._model, self._renderer_thread, GetMetricFromMetricType)
93    self._ps = page_set.PageSet(file_path=os.path.dirname(__file__))
94    self._ps.AddPageWithDefaultRunNavigate('http://www.bar.com/')
95    self._results.WillRunPage(self._ps.pages[0])
96
97  def AddResults(self):
98    self._metric.AddResults(self._results)
99    self._results.DidRunPage(self._ps.pages[0])
100
101
102class TimelineBasedMetricsTests(unittest.TestCase):
103
104  def testFindTimelineInteractionRecords(self):
105    d = TimelineBasedMetricTestData()
106    d.AddInteraction(ts=0, duration=20,
107                     marker='Interaction.LogicalName1/is_smooth')
108    d.AddInteraction(ts=25, duration=5,
109                     marker='Interaction.LogicalName2/is_responsive')
110    d.AddInteraction(ts=50, duration=15,
111                     marker='Interaction.LogicalName3/is_fast')
112    d.FinalizeImport()
113    interactions = d.metric.FindTimelineInteractionRecords()
114    self.assertEquals(3, len(interactions))
115    self.assertTrue(interactions[0].is_smooth)
116    self.assertEquals(0, interactions[0].start)
117    self.assertEquals(20, interactions[0].end)
118
119    self.assertTrue(interactions[1].is_responsive)
120    self.assertEquals(25, interactions[1].start)
121    self.assertEquals(30, interactions[1].end)
122
123    self.assertTrue(interactions[2].is_fast)
124    self.assertEquals(50, interactions[2].start)
125    self.assertEquals(65, interactions[2].end)
126
127  def testAddResults(self):
128    d = TimelineBasedMetricTestData()
129    d.AddInteraction(ts=0, duration=20,
130                     marker='Interaction.LogicalName1/is_smooth')
131    d.AddInteraction(ts=25, duration=5,
132                     marker='Interaction.LogicalName2/is_responsive')
133    d.AddInteraction(ts=50, duration=15,
134                     marker='Interaction.LogicalName3/is_fast')
135    d.FinalizeImport()
136    d.AddResults()
137    self.assertEquals(1, len(d.results.FindAllPageSpecificValuesNamed(
138        'LogicalName1-FakeSmoothMetric')))
139    self.assertEquals(1, len(d.results.FindAllPageSpecificValuesNamed(
140        'LogicalName2-FakeLoadingMetric')))
141    self.assertEquals(1, len(d.results.FindAllPageSpecificValuesNamed(
142        'LogicalName3-FakeFastMetric')))
143
144  def testNoInteractions(self):
145    d = TimelineBasedMetricTestData()
146    d.FinalizeImport()
147    self.assertRaises(tbm_module.InvalidInteractions, d.AddResults)
148
149  def testDuplicateUnrepeatableInteractions(self):
150    d = TimelineBasedMetricTestData()
151    d.AddInteraction(ts=10, duration=5,
152                     marker='Interaction.LogicalName/is_smooth')
153    d.AddInteraction(ts=20, duration=5,
154                     marker='Interaction.LogicalName/is_smooth')
155    d.FinalizeImport()
156    self.assertRaises(tbm_module.InvalidInteractions, d.AddResults)
157
158  def testDuplicateRepeatableInteractions(self):
159    d = TimelineBasedMetricTestData()
160    d.AddInteraction(ts=10, duration=5,
161                     marker='Interaction.LogicalName/is_smooth,repeatable')
162    d.AddInteraction(ts=20, duration=5,
163                     marker='Interaction.LogicalName/is_smooth,repeatable')
164    d.FinalizeImport()
165    d.AddResults()
166    self.assertEquals(1, len(d.results.pages_that_succeeded))
167
168  def testDuplicateRepeatableInteractionsWithDifferentMetrics(self):
169    d = TimelineBasedMetricTestData()
170
171    responsive_marker = 'Interaction.LogicalName/is_responsive,repeatable'
172    d.AddInteraction(ts=10, duration=5, marker=responsive_marker)
173    smooth_marker = 'Interaction.LogicalName/is_smooth,repeatable'
174    d.AddInteraction(ts=20, duration=5, marker=smooth_marker)
175    d.FinalizeImport()
176    self.assertRaises(tbm_module.InvalidInteractions, d.AddResults)
177
178
179class TestTimelinebasedMeasurementPage(page_module.Page):
180
181  def __init__(self, ps, base_dir, trigger_animation=False,
182               trigger_jank=False, trigger_slow=False):
183    super(TestTimelinebasedMeasurementPage, self).__init__(
184        'file://interaction_enabled_page.html', ps, base_dir)
185    self._trigger_animation = trigger_animation
186    self._trigger_jank = trigger_jank
187    self._trigger_slow = trigger_slow
188
189  def RunSmoothness(self, action_runner):
190    if self._trigger_animation:
191      action_runner.TapElement('#animating-button')
192      action_runner.WaitForJavaScriptCondition('window.animationDone')
193    if self._trigger_jank:
194      action_runner.TapElement('#jank-button')
195      action_runner.WaitForJavaScriptCondition('window.jankScriptDone')
196    if self._trigger_slow:
197      action_runner.TapElement('#slow-button')
198      action_runner.WaitForJavaScriptCondition('window.slowScriptDone')
199
200
201class TimelineBasedMeasurementTest(page_test_test_case.PageTestTestCase):
202
203  def setUp(self):
204    self._options = options_for_unittests.GetCopy()
205    self._options.browser_options.wpr_mode = wpr_modes.WPR_OFF
206
207  def testSmoothnessTimelineBasedMeasurementForSmoke(self):
208    ps = self.CreateEmptyPageSet()
209    ps.AddPage(TestTimelinebasedMeasurementPage(
210        ps, ps.base_dir, trigger_animation=True))
211
212    measurement = tbm_module.TimelineBasedMeasurement()
213    results = self.RunMeasurement(measurement, ps,
214                                  options=self._options)
215
216    self.assertEquals(0, len(results.failures))
217    v = results.FindAllPageSpecificValuesNamed('CenterAnimation-jank')
218    self.assertEquals(len(v), 1)
219    v = results.FindAllPageSpecificValuesNamed('DrawerAnimation-jank')
220    self.assertEquals(len(v), 1)
221
222  def testFastTimelineBasedMeasurementForSmoke(self):
223    ps = self.CreateEmptyPageSet()
224    ps.AddPage(TestTimelinebasedMeasurementPage(
225        ps, ps.base_dir, trigger_slow=True))
226
227    measurement = tbm_module.TimelineBasedMeasurement()
228    results = self.RunMeasurement(measurement, ps, options=self._options)
229
230    self.assertEquals([], results.failures)
231    expected_names = set([
232        'SlowThreadJsRun-fast-duration',
233        'SlowThreadJsRun-fast-idle_time',
234        'SlowThreadJsRun-fast-incremental_marking',
235        'SlowThreadJsRun-fast-incremental_marking_outside_idle',
236        'SlowThreadJsRun-fast-mark_compactor',
237        'SlowThreadJsRun-fast-mark_compactor_outside_idle',
238        'SlowThreadJsRun-fast-scavenger',
239        'SlowThreadJsRun-fast-scavenger_outside_idle',
240        'SlowThreadJsRun-fast-total_garbage_collection',
241        'SlowThreadJsRun-fast-total_garbage_collection_outside_idle',
242        ])
243    if platform.GetHostPlatform().GetOSName() != 'win':
244      # CPU metric is only supported non-Windows platforms.
245      expected_names.add('SlowThreadJsRun-fast-cpu_time')
246    self.assertEquals(
247        expected_names, set(v.name for v in results.all_page_specific_values))
248
249    # In interaction_enabled_page.html, the "slow" interaction executes
250    # a loop with window.performance.now() to wait 200ms.
251    # fast-duration measures wall time so its value should be at least 200ms.
252    v = results.FindAllPageSpecificValuesNamed('SlowThreadJsRun-fast-duration')
253    self.assertGreaterEqual(v[0].value, 200.0)
254
255  # Disabled since mainthread_jank metric is not supported on windows platform.
256  @benchmark.Disabled('win')
257  def testMainthreadJankTimelineBasedMeasurement(self):
258    ps = self.CreateEmptyPageSet()
259    ps.AddPage(TestTimelinebasedMeasurementPage(
260        ps, ps.base_dir, trigger_jank=True))
261
262    measurement = tbm_module.TimelineBasedMeasurement()
263    results = self.RunMeasurement(measurement, ps,
264                                  options=self._options)
265    self.assertEquals(0, len(results.failures))
266
267    # In interaction_enabled_page.html, we create a jank loop based on
268    # window.performance.now() (basically loop for x milliseconds).
269    # Since window.performance.now() uses wall-time instead of thread time,
270    # we only assert the biggest jank > 50ms here to account for the fact
271    # that the browser may deschedule during the jank loop.
272    v = results.FindAllPageSpecificValuesNamed(
273        'JankThreadJSRun-responsive-biggest_jank_thread_time')
274    self.assertGreaterEqual(v[0].value, 50)
275
276    v = results.FindAllPageSpecificValuesNamed(
277        'JankThreadJSRun-responsive-total_big_jank_thread_time')
278    self.assertGreaterEqual(v[0].value, 50)
279