1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import math
6import os
7import unittest
8
9from telemetry import story
10from telemetry.internal.results import page_test_results
11from telemetry import page as page_module
12from telemetry.value import failure
13from telemetry.value import histogram
14from telemetry.value import improvement_direction
15from telemetry.value import list_of_scalar_values
16from telemetry.value import scalar
17from telemetry.value import summary as summary_module
18
19
20class TestBase(unittest.TestCase):
21  def setUp(self):
22    story_set = story.StorySet(base_dir=os.path.dirname(__file__))
23    story_set.AddStory(
24        page_module.Page('http://www.bar.com/', story_set, story_set.base_dir))
25    story_set.AddStory(
26        page_module.Page('http://www.baz.com/', story_set, story_set.base_dir))
27    story_set.AddStory(
28        page_module.Page('http://www.foo.com/', story_set, story_set.base_dir))
29    self.story_set = story_set
30
31  @property
32  def pages(self):
33    return self.story_set.stories
34
35
36class SummaryTest(TestBase):
37  def testBasicSummary(self):
38    page0 = self.pages[0]
39    page1 = self.pages[1]
40
41    results = page_test_results.PageTestResults()
42
43    results.WillRunPage(page0)
44    v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3,
45                            improvement_direction=improvement_direction.UP)
46    results.AddValue(v0)
47    results.DidRunPage(page0)
48
49    results.WillRunPage(page1)
50    v1 = scalar.ScalarValue(page1, 'a', 'seconds', 7,
51                            improvement_direction=improvement_direction.UP)
52    results.AddValue(v1)
53    results.DidRunPage(page1)
54
55    summary = summary_module.Summary(results.all_page_specific_values)
56    values = summary.interleaved_computed_per_page_values_and_summaries
57
58    v0_list = list_of_scalar_values.ListOfScalarValues(
59        page0, 'a', 'seconds', [3],
60        improvement_direction=improvement_direction.UP)
61    v1_list = list_of_scalar_values.ListOfScalarValues(
62        page1, 'a', 'seconds', [7],
63        improvement_direction=improvement_direction.UP)
64    # Std is 0 because we only have one measurement per page.
65    merged_value = list_of_scalar_values.ListOfScalarValues(
66        None, 'a', 'seconds', [3, 7], std=0.0,
67        improvement_direction=improvement_direction.UP)
68
69    self.assertEquals(3, len(values))
70    self.assertIn(v0_list, values)
71    self.assertIn(v1_list, values)
72    self.assertIn(merged_value, values)
73
74  def testBasicSummaryWithOnlyOnePage(self):
75    page0 = self.pages[0]
76
77    results = page_test_results.PageTestResults()
78
79    results.WillRunPage(page0)
80    v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3,
81                            improvement_direction=improvement_direction.UP)
82    results.AddValue(v0)
83    results.DidRunPage(page0)
84
85    summary = summary_module.Summary(results.all_page_specific_values)
86    values = summary.interleaved_computed_per_page_values_and_summaries
87
88    v0_list = list_of_scalar_values.ListOfScalarValues(
89        page0, 'a', 'seconds', [3],
90        improvement_direction=improvement_direction.UP)
91    merged_list = list_of_scalar_values.ListOfScalarValues(
92        None, 'a', 'seconds', [3],
93        improvement_direction=improvement_direction.UP)
94
95    self.assertEquals(2, len(values))
96    self.assertIn(v0_list, values)
97    self.assertIn(merged_list, values)
98
99  def testBasicSummaryNonuniformResults(self):
100    page0 = self.pages[0]
101    page1 = self.pages[1]
102    page2 = self.pages[2]
103
104    results = page_test_results.PageTestResults()
105    results.WillRunPage(page0)
106    v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3,
107                            improvement_direction=improvement_direction.UP)
108    results.AddValue(v0)
109    v1 = scalar.ScalarValue(page0, 'b', 'seconds', 10,
110                            improvement_direction=improvement_direction.UP)
111    results.AddValue(v1)
112    results.DidRunPage(page0)
113
114    results.WillRunPage(page1)
115    v2 = scalar.ScalarValue(page1, 'a', 'seconds', 3,
116                            improvement_direction=improvement_direction.UP)
117    results.AddValue(v2)
118    v3 = scalar.ScalarValue(page1, 'b', 'seconds', 10,
119                            improvement_direction=improvement_direction.UP)
120    results.AddValue(v3)
121    results.DidRunPage(page1)
122
123    results.WillRunPage(page2)
124    v4 = scalar.ScalarValue(page2, 'a', 'seconds', 7,
125                            improvement_direction=improvement_direction.UP)
126    results.AddValue(v4)
127    # Note, page[2] does not report a 'b' metric.
128    results.DidRunPage(page2)
129
130    summary = summary_module.Summary(results.all_page_specific_values)
131    values = summary.interleaved_computed_per_page_values_and_summaries
132
133    v0_list = list_of_scalar_values.ListOfScalarValues(
134        page0, 'a', 'seconds', [3],
135        improvement_direction=improvement_direction.UP)
136    v1_list = list_of_scalar_values.ListOfScalarValues(
137        page0, 'b', 'seconds', [10],
138        improvement_direction=improvement_direction.UP)
139    v2_list = list_of_scalar_values.ListOfScalarValues(
140        page1, 'a', 'seconds', [3],
141        improvement_direction=improvement_direction.UP)
142    v3_list = list_of_scalar_values.ListOfScalarValues(
143        page1, 'b', 'seconds', [10],
144        improvement_direction=improvement_direction.UP)
145    v4_list = list_of_scalar_values.ListOfScalarValues(
146        page2, 'a', 'seconds', [7],
147        improvement_direction=improvement_direction.UP)
148    # Std is 0 because we only have one measurement per page.
149    a_summary = list_of_scalar_values.ListOfScalarValues(
150        None, 'a', 'seconds', [3, 3, 7], std=0.0,
151        improvement_direction=improvement_direction.UP)
152    b_summary = list_of_scalar_values.ListOfScalarValues(
153        None, 'b', 'seconds', [10, 10], std=0.0,
154        improvement_direction=improvement_direction.UP)
155
156    self.assertEquals(7, len(values))
157    self.assertIn(v0_list, values)
158    self.assertIn(v1_list, values)
159    self.assertIn(v2_list, values)
160    self.assertIn(v3_list, values)
161    self.assertIn(v4_list, values)
162    self.assertIn(a_summary, values)
163    self.assertIn(b_summary, values)
164
165  def testBasicSummaryPassAndFailPage(self):
166    """If a page failed, only print summary for individual pages."""
167    page0 = self.pages[0]
168    page1 = self.pages[1]
169
170    results = page_test_results.PageTestResults()
171    results.WillRunPage(page0)
172    v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3,
173                            improvement_direction=improvement_direction.UP)
174    results.AddValue(v0)
175    v1 = failure.FailureValue.FromMessage(page0, 'message')
176    results.AddValue(v1)
177    results.DidRunPage(page0)
178
179    results.WillRunPage(page1)
180    v2 = scalar.ScalarValue(page1, 'a', 'seconds', 7,
181                            improvement_direction=improvement_direction.UP)
182    results.AddValue(v2)
183    results.DidRunPage(page1)
184
185    summary = summary_module.Summary(results.all_page_specific_values)
186    values = summary.interleaved_computed_per_page_values_and_summaries
187
188    v0_list = list_of_scalar_values.ListOfScalarValues(
189        page0, 'a', 'seconds', [3],
190        improvement_direction=improvement_direction.UP)
191    v2_list = list_of_scalar_values.ListOfScalarValues(
192        page1, 'a', 'seconds', [7],
193        improvement_direction=improvement_direction.UP)
194
195    self.assertEquals(2, len(values))
196    self.assertIn(v0_list, values)
197    self.assertIn(v2_list, values)
198
199  def testRepeatedPagesetOneIterationOnePageFails(self):
200    """Page fails on one iteration, no averaged results should print."""
201    page0 = self.pages[0]
202    page1 = self.pages[1]
203
204    results = page_test_results.PageTestResults()
205    results.WillRunPage(page0)
206    v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3,
207                            improvement_direction=improvement_direction.UP)
208    results.AddValue(v0)
209    results.DidRunPage(page0)
210
211    results.WillRunPage(page1)
212    v1 = scalar.ScalarValue(page1, 'a', 'seconds', 7,
213                            improvement_direction=improvement_direction.UP)
214    results.AddValue(v1)
215    v2 = failure.FailureValue.FromMessage(page1, 'message')
216    results.AddValue(v2)
217    results.DidRunPage(page1)
218
219    results.WillRunPage(page0)
220    v3 = scalar.ScalarValue(page0, 'a', 'seconds', 4,
221                            improvement_direction=improvement_direction.UP)
222    results.AddValue(v3)
223    results.DidRunPage(page0)
224
225    results.WillRunPage(page1)
226    v4 = scalar.ScalarValue(page1, 'a', 'seconds', 8,
227                            improvement_direction=improvement_direction.UP)
228    results.AddValue(v4)
229    results.DidRunPage(page1)
230
231    summary = summary_module.Summary(results.all_page_specific_values)
232    values = summary.interleaved_computed_per_page_values_and_summaries
233
234    page0_aggregated = list_of_scalar_values.ListOfScalarValues(
235        page0, 'a', 'seconds', [3, 4],
236        improvement_direction=improvement_direction.UP)
237    page1_aggregated = list_of_scalar_values.ListOfScalarValues(
238        page1, 'a', 'seconds', [7, 8],
239        improvement_direction=improvement_direction.UP)
240
241    self.assertEquals(2, len(values))
242    self.assertIn(page0_aggregated, values)
243    self.assertIn(page1_aggregated, values)
244
245  def testRepeatedPages(self):
246    page0 = self.pages[0]
247    page1 = self.pages[1]
248
249    results = page_test_results.PageTestResults()
250    results.WillRunPage(page0)
251    v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3,
252                            improvement_direction=improvement_direction.UP)
253    results.AddValue(v0)
254    results.DidRunPage(page0)
255
256    results.WillRunPage(page0)
257    v2 = scalar.ScalarValue(page0, 'a', 'seconds', 4,
258                            improvement_direction=improvement_direction.UP)
259    results.AddValue(v2)
260    results.DidRunPage(page0)
261
262    results.WillRunPage(page1)
263    v1 = scalar.ScalarValue(page1, 'a', 'seconds', 7,
264                            improvement_direction=improvement_direction.UP)
265    results.AddValue(v1)
266    results.DidRunPage(page1)
267
268    results.WillRunPage(page1)
269    v3 = scalar.ScalarValue(page1, 'a', 'seconds', 8,
270                            improvement_direction=improvement_direction.UP)
271    results.AddValue(v3)
272    results.DidRunPage(page1)
273
274    summary = summary_module.Summary(results.all_page_specific_values)
275    values = summary.interleaved_computed_per_page_values_and_summaries
276
277    page0_aggregated = list_of_scalar_values.ListOfScalarValues(
278        page0, 'a', 'seconds', [3, 4],
279        improvement_direction=improvement_direction.UP)
280    page1_aggregated = list_of_scalar_values.ListOfScalarValues(
281        page1, 'a', 'seconds', [7, 8],
282        improvement_direction=improvement_direction.UP)
283    # Std is computed using pooled standard deviation.
284    a_summary = list_of_scalar_values.ListOfScalarValues(
285        None, 'a', 'seconds', [3, 4, 7, 8], std=math.sqrt(0.5),
286        improvement_direction=improvement_direction.UP)
287
288    self.assertEquals(3, len(values))
289    self.assertIn(page0_aggregated, values)
290    self.assertIn(page1_aggregated, values)
291    self.assertIn(a_summary, values)
292
293  def testPageRunsTwice(self):
294    page0 = self.pages[0]
295
296    results = page_test_results.PageTestResults()
297
298    results.WillRunPage(page0)
299    v0 = scalar.ScalarValue(page0, 'b', 'seconds', 2,
300                            improvement_direction=improvement_direction.UP)
301    results.AddValue(v0)
302    results.DidRunPage(page0)
303
304    results.WillRunPage(page0)
305    v1 = scalar.ScalarValue(page0, 'b', 'seconds', 3,
306                            improvement_direction=improvement_direction.UP)
307    results.AddValue(v1)
308    results.DidRunPage(page0)
309
310    summary = summary_module.Summary(results.all_page_specific_values)
311    values = summary.interleaved_computed_per_page_values_and_summaries
312
313    page0_aggregated = list_of_scalar_values.ListOfScalarValues(
314        page0, 'b', 'seconds', [2, 3],
315        improvement_direction=improvement_direction.UP)
316    b_summary = list_of_scalar_values.ListOfScalarValues(
317        None, 'b', 'seconds', [2, 3],
318        improvement_direction=improvement_direction.UP)
319
320    self.assertEquals(2, len(values))
321    self.assertIn(page0_aggregated, values)
322    self.assertIn(b_summary, values)
323
324  def testListValue(self):
325    page0 = self.pages[0]
326    page1 = self.pages[1]
327
328    results = page_test_results.PageTestResults()
329
330    results.WillRunPage(page0)
331    v0 = list_of_scalar_values.ListOfScalarValues(
332        page0, 'b', 'seconds', [2, 2],
333        improvement_direction=improvement_direction.UP)
334    results.AddValue(v0)
335    results.DidRunPage(page0)
336
337    results.WillRunPage(page1)
338    v1 = list_of_scalar_values.ListOfScalarValues(
339        page1, 'b', 'seconds', [3, 3],
340        improvement_direction=improvement_direction.UP)
341    results.AddValue(v1)
342    results.DidRunPage(page1)
343
344    summary = summary_module.Summary(results.all_page_specific_values)
345    values = summary.interleaved_computed_per_page_values_and_summaries
346
347    b_summary = list_of_scalar_values.ListOfScalarValues(
348        None, 'b', 'seconds', [2, 2, 3, 3], std=0.0,
349        improvement_direction=improvement_direction.UP)
350
351    self.assertEquals(3, len(values))
352    self.assertIn(v0, values)
353    self.assertIn(v1, values)
354    self.assertIn(b_summary, values)
355
356  def testHistogram(self):
357    page0 = self.pages[0]
358    page1 = self.pages[1]
359
360    results = page_test_results.PageTestResults()
361    results.WillRunPage(page0)
362    v0 = histogram.HistogramValue(
363        page0, 'a', 'units',
364        raw_value_json='{"buckets": [{"low": 1, "high": 2, "count": 1}]}',
365        important=False, improvement_direction=improvement_direction.UP)
366    results.AddValue(v0)
367    results.DidRunPage(page0)
368
369    results.WillRunPage(page1)
370    v1 = histogram.HistogramValue(
371        page1, 'a', 'units',
372        raw_value_json='{"buckets": [{"low": 2, "high": 3, "count": 1}]}',
373        important=False, improvement_direction=improvement_direction.UP)
374    results.AddValue(v1)
375    results.DidRunPage(page1)
376
377    summary = summary_module.Summary(results.all_page_specific_values)
378    values = summary.interleaved_computed_per_page_values_and_summaries
379
380    self.assertEquals(2, len(values))
381    self.assertIn(v0, values)
382    self.assertIn(v1, values)
383
384  def testSummaryUsesKeyFunc(self):
385    page0 = self.pages[0]
386    page1 = self.pages[1]
387
388    results = page_test_results.PageTestResults()
389
390    results.WillRunPage(page0)
391    v0 = scalar.ScalarValue(page0, 'a', 'seconds', 20,
392                            improvement_direction=improvement_direction.UP)
393    results.AddValue(v0)
394
395    v1 = scalar.ScalarValue(page0, 'b', 'seconds', 42,
396                            improvement_direction=improvement_direction.UP)
397    results.AddValue(v1)
398    results.DidRunPage(page0)
399
400    results.WillRunPage(page1)
401    v2 = scalar.ScalarValue(page1, 'a', 'seconds', 20,
402                            improvement_direction=improvement_direction.UP)
403    results.AddValue(v2)
404
405    v3 = scalar.ScalarValue(page1, 'b', 'seconds', 42,
406                            improvement_direction=improvement_direction.UP)
407    results.AddValue(v3)
408    results.DidRunPage(page1)
409
410    summary = summary_module.Summary(
411        results.all_page_specific_values,
412        key_func=lambda v: True)
413    values = summary.interleaved_computed_per_page_values_and_summaries
414
415    v0_list = list_of_scalar_values.ListOfScalarValues(
416        page0, 'a', 'seconds', [20, 42],
417        improvement_direction=improvement_direction.UP)
418    v2_list = list_of_scalar_values.ListOfScalarValues(
419        page1, 'a', 'seconds', [20, 42],
420        improvement_direction=improvement_direction.UP)
421    # Std is computed using pooled standard deviation.
422    merged_value = list_of_scalar_values.ListOfScalarValues(
423        None, 'a', 'seconds', [20, 42, 20, 42], std=math.sqrt(242.0),
424        improvement_direction=improvement_direction.UP)
425
426    self.assertEquals(3, len(values))
427    self.assertIn(v0_list, values)
428    self.assertIn(v2_list, values)
429    self.assertIn(merged_value, values)
430