1# Copyright (c) 2012 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4import os
5import unittest
6
7from telemetry.page import buildbot_page_measurement_results
8from telemetry.page import page_set
9from telemetry.page import perf_tests_helper
10
11def _MakePageSet():
12  return page_set.PageSet.FromDict({
13      "description": "hello",
14      "archive_path": "foo.wpr",
15      "pages": [
16        {"url": "http://www.foo.com/"},
17        {"url": "http://www.bar.com/"},
18        {"url": "http://www.baz.com/"}
19        ]
20      }, os.path.dirname(__file__))
21
22class SummarySavingPageMeasurementResults(
23    buildbot_page_measurement_results.BuildbotPageMeasurementResults):
24  def __init__(self, trace_tag=''):
25    super(SummarySavingPageMeasurementResults, self).__init__(trace_tag)
26    self.results = []
27
28  def _PrintPerfResult(self, *args):
29    res = perf_tests_helper.PrintPerfResult(*args, print_to_stdout=False)
30    self.results.append(res)
31
32class BuildbotPageMeasurementResultsTest(unittest.TestCase):
33  def test_basic_summary(self):
34    test_page_set = _MakePageSet()
35
36    measurement_results = SummarySavingPageMeasurementResults()
37    measurement_results.WillMeasurePage(test_page_set.pages[0])
38    measurement_results.Add('a', 'seconds', 3)
39    measurement_results.DidMeasurePage()
40
41    measurement_results.WillMeasurePage(test_page_set.pages[1])
42    measurement_results.Add('a', 'seconds', 7)
43    measurement_results.DidMeasurePage()
44
45    measurement_results.PrintSummary()
46    expected = ['RESULT a_by_url: http___www.foo.com_= 3 seconds',
47                'RESULT a_by_url: http___www.bar.com_= 7 seconds',
48                '*RESULT a: a= [3,7] seconds\nAvg a: 5.000000seconds\n' +
49                'Sd  a: 2.828427seconds']
50    self.assertEquals(measurement_results.results, expected)
51
52  def test_basic_summary_nonuniform_results(self):
53    test_page_set = _MakePageSet()
54
55    measurement_results = SummarySavingPageMeasurementResults()
56    measurement_results.WillMeasurePage(test_page_set.pages[0])
57    measurement_results.Add('a', 'seconds', 3)
58    measurement_results.Add('b', 'seconds', 10)
59    measurement_results.DidMeasurePage()
60
61    measurement_results.WillMeasurePage(test_page_set.pages[1])
62    measurement_results.Add('a', 'seconds', 3)
63    measurement_results.Add('b', 'seconds', 10)
64    measurement_results.DidMeasurePage()
65
66    measurement_results.WillMeasurePage(test_page_set.pages[2])
67    measurement_results.Add('a', 'seconds', 7)
68    # Note, page[2] does not report a 'b' metric.
69    measurement_results.DidMeasurePage()
70
71    measurement_results.PrintSummary()
72    expected = ['RESULT a_by_url: http___www.foo.com_= 3 seconds',
73                'RESULT a_by_url: http___www.bar.com_= 3 seconds',
74                'RESULT a_by_url: http___www.baz.com_= 7 seconds',
75                '*RESULT a: a= [3,3,7] seconds\nAvg a: 4.333333seconds\n' +
76                    'Sd  a: 2.309401seconds',
77                'RESULT b_by_url: http___www.foo.com_= 10 seconds',
78                'RESULT b_by_url: http___www.bar.com_= 10 seconds',
79                '*RESULT b: b= [10,10] seconds\nAvg b: 10.000000seconds']
80    self.assertEquals(measurement_results.results, expected)
81
82  def test_basic_summary_pass_and_fail_page(self):
83    """If a page failed, only print summary for individual passing pages."""
84    test_page_set = _MakePageSet()
85
86    measurement_results = SummarySavingPageMeasurementResults()
87    measurement_results.WillMeasurePage(test_page_set.pages[0])
88    measurement_results.Add('a', 'seconds', 3)
89    measurement_results.DidMeasurePage()
90    measurement_results.AddFailureMessage(test_page_set.pages[0], 'message')
91
92    measurement_results.WillMeasurePage(test_page_set.pages[1])
93    measurement_results.Add('a', 'seconds', 7)
94    measurement_results.DidMeasurePage()
95
96    measurement_results.PrintSummary()
97    expected = ['RESULT a_by_url: http___www.bar.com_= 7 seconds']
98    self.assertEquals(measurement_results.results, expected)
99
100  def test_repeated_pageset_one_iteration_one_page_fails(self):
101    """Page fails on one iteration, no results for that page should print."""
102    test_page_set = _MakePageSet()
103
104    measurement_results = SummarySavingPageMeasurementResults()
105    measurement_results.WillMeasurePage(test_page_set.pages[0])
106    measurement_results.Add('a', 'seconds', 3)
107    measurement_results.DidMeasurePage()
108
109    measurement_results.WillMeasurePage(test_page_set.pages[1])
110    measurement_results.Add('a', 'seconds', 7)
111    measurement_results.DidMeasurePage()
112    measurement_results.AddFailureMessage(test_page_set.pages[1], 'message')
113
114    measurement_results.WillMeasurePage(test_page_set.pages[0])
115    measurement_results.Add('a', 'seconds', 4)
116    measurement_results.DidMeasurePage()
117
118    measurement_results.WillMeasurePage(test_page_set.pages[1])
119    measurement_results.Add('a', 'seconds', 8)
120    measurement_results.DidMeasurePage()
121
122    measurement_results.PrintSummary()
123    expected = ['RESULT a_by_url: http___www.foo.com_= [3,4] seconds\n' +
124                'Avg a_by_url: 3.500000seconds\nSd  a_by_url: 0.707107seconds']
125    self.assertEquals(measurement_results.results, expected)
126
127  def test_repeated_pageset(self):
128    test_page_set = _MakePageSet()
129
130    measurement_results = SummarySavingPageMeasurementResults()
131    measurement_results.WillMeasurePage(test_page_set.pages[0])
132    measurement_results.Add('a', 'seconds', 3)
133    measurement_results.DidMeasurePage()
134
135    measurement_results.WillMeasurePage(test_page_set.pages[1])
136    measurement_results.Add('a', 'seconds', 7)
137    measurement_results.DidMeasurePage()
138
139    measurement_results.WillMeasurePage(test_page_set.pages[0])
140    measurement_results.Add('a', 'seconds', 4)
141    measurement_results.DidMeasurePage()
142
143    measurement_results.WillMeasurePage(test_page_set.pages[1])
144    measurement_results.Add('a', 'seconds', 8)
145    measurement_results.DidMeasurePage()
146
147    measurement_results.PrintSummary()
148    expected = ['RESULT a_by_url: http___www.foo.com_= [3,4] seconds\n' +
149                'Avg a_by_url: 3.500000seconds\nSd  a_by_url: 0.707107seconds',
150                'RESULT a_by_url: http___www.bar.com_= [7,8] seconds\n' +
151                'Avg a_by_url: 7.500000seconds\nSd  a_by_url: 0.707107seconds',
152                '*RESULT a: a= [3,7,4,8] seconds\n' +
153                'Avg a: 5.500000seconds\nSd  a: 2.380476seconds'
154                ]
155    self.assertEquals(
156      measurement_results.results,
157      expected)
158
159  def test_repeated_pages(self):
160    test_page_set = _MakePageSet()
161
162    measurement_results = SummarySavingPageMeasurementResults()
163    measurement_results.WillMeasurePage(test_page_set.pages[0])
164    measurement_results.Add('a', 'seconds', 3)
165    measurement_results.DidMeasurePage()
166
167    measurement_results.WillMeasurePage(test_page_set.pages[0])
168    measurement_results.Add('a', 'seconds', 4)
169    measurement_results.DidMeasurePage()
170
171    measurement_results.WillMeasurePage(test_page_set.pages[1])
172    measurement_results.Add('a', 'seconds', 7)
173    measurement_results.DidMeasurePage()
174
175    measurement_results.WillMeasurePage(test_page_set.pages[1])
176    measurement_results.Add('a', 'seconds', 8)
177    measurement_results.DidMeasurePage()
178
179    measurement_results.PrintSummary()
180    expected = ['RESULT a_by_url: http___www.foo.com_= [3,4] seconds\n' +
181                'Avg a_by_url: 3.500000seconds\nSd  a_by_url: 0.707107seconds',
182                'RESULT a_by_url: http___www.bar.com_= [7,8] seconds\n' +
183                'Avg a_by_url: 7.500000seconds\nSd  a_by_url: 0.707107seconds',
184                '*RESULT a: a= [3,4,7,8] seconds\n' +
185                'Avg a: 5.500000seconds\nSd  a: 2.380476seconds'
186                ]
187    self.assertEquals(
188      measurement_results.results,
189      expected)
190
191  def test_overall_results(self):
192    test_page_set = _MakePageSet()
193
194    measurement_results = SummarySavingPageMeasurementResults()
195
196    measurement_results.AddSummary('a', 'seconds', 1)
197
198    measurement_results.WillMeasurePage(test_page_set.pages[0])
199    measurement_results.Add('b', 'seconds', 2)
200    measurement_results.DidMeasurePage()
201
202    measurement_results.WillMeasurePage(test_page_set.pages[1])
203    measurement_results.Add('b', 'seconds', 3)
204    measurement_results.DidMeasurePage()
205
206    measurement_results.AddSummary('c', 'seconds', 4)
207
208    measurement_results.PrintSummary()
209
210    self.assertEquals(
211        measurement_results.results,
212        ['RESULT b_by_url: http___www.foo.com_= 2 seconds',
213         'RESULT b_by_url: http___www.bar.com_= 3 seconds',
214         '*RESULT b: b= [2,3] seconds\n' +
215         'Avg b: 2.500000seconds\nSd  b: 0.707107seconds',
216         '*RESULT a: a= 1 seconds',
217         '*RESULT c: c= 4 seconds'])
218
219  def test_overall_results_trace_tag(self):
220    test_page_set = _MakePageSet()
221
222    measurement_results = SummarySavingPageMeasurementResults(trace_tag='_ref')
223
224    measurement_results.AddSummary('a', 'seconds', 1)
225
226    measurement_results.WillMeasurePage(test_page_set.pages[0])
227    measurement_results.Add('b', 'seconds', 2)
228    measurement_results.DidMeasurePage()
229
230    measurement_results.WillMeasurePage(test_page_set.pages[1])
231    measurement_results.Add('b', 'seconds', 3)
232    measurement_results.DidMeasurePage()
233
234    measurement_results.AddSummary('c', 'seconds', 4)
235
236    measurement_results.PrintSummary()
237
238    self.assertEquals(
239        measurement_results.results,
240        ['*RESULT b: b_ref= [2,3] seconds\n' +
241         'Avg b: 2.500000seconds\nSd  b: 0.707107seconds',
242         '*RESULT a: a_ref= 1 seconds',
243         '*RESULT c: c_ref= 4 seconds'])
244
245  def test_histogram(self):
246    test_page_set = _MakePageSet()
247
248    measurement_results = SummarySavingPageMeasurementResults()
249    measurement_results.WillMeasurePage(test_page_set.pages[0])
250    measurement_results.Add('a', '',
251                          '{"buckets": [{"low": 1, "high": 2, "count": 1}]}',
252                          data_type='histogram')
253    measurement_results.DidMeasurePage()
254
255    measurement_results.WillMeasurePage(test_page_set.pages[1])
256    measurement_results.Add('a', '',
257                          '{"buckets": [{"low": 2, "high": 3, "count": 1}]}',
258                          data_type='histogram')
259    measurement_results.DidMeasurePage()
260
261    measurement_results.PrintSummary()
262
263    expected = [
264        'HISTOGRAM a_by_url: http___www.foo.com_= ' +
265        '{"buckets": [{"low": 1, "high": 2, "count": 1}]}\n' +
266        'Avg a_by_url: 1.500000',
267        'HISTOGRAM a_by_url: http___www.bar.com_= ' +
268        '{"buckets": [{"low": 2, "high": 3, "count": 1}]}\n' +
269        'Avg a_by_url: 2.500000']
270    self.assertEquals(measurement_results.results, expected)
271