1# Copyright (c) 2012 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4import os 5import unittest 6 7from telemetry.page import buildbot_page_measurement_results 8from telemetry.page import page_set 9from telemetry.page import perf_tests_helper 10 11def _MakePageSet(): 12 return page_set.PageSet.FromDict({ 13 "description": "hello", 14 "archive_path": "foo.wpr", 15 "pages": [ 16 {"url": "http://www.foo.com/"}, 17 {"url": "http://www.bar.com/"}, 18 {"url": "http://www.baz.com/"} 19 ] 20 }, os.path.dirname(__file__)) 21 22class SummarySavingPageMeasurementResults( 23 buildbot_page_measurement_results.BuildbotPageMeasurementResults): 24 def __init__(self, trace_tag=''): 25 super(SummarySavingPageMeasurementResults, self).__init__(trace_tag) 26 self.results = [] 27 28 def _PrintPerfResult(self, *args): 29 res = perf_tests_helper.PrintPerfResult(*args, print_to_stdout=False) 30 self.results.append(res) 31 32class BuildbotPageMeasurementResultsTest(unittest.TestCase): 33 def test_basic_summary(self): 34 test_page_set = _MakePageSet() 35 36 measurement_results = SummarySavingPageMeasurementResults() 37 measurement_results.WillMeasurePage(test_page_set.pages[0]) 38 measurement_results.Add('a', 'seconds', 3) 39 measurement_results.DidMeasurePage() 40 41 measurement_results.WillMeasurePage(test_page_set.pages[1]) 42 measurement_results.Add('a', 'seconds', 7) 43 measurement_results.DidMeasurePage() 44 45 measurement_results.PrintSummary() 46 expected = ['RESULT a_by_url: http___www.foo.com_= 3 seconds', 47 'RESULT a_by_url: http___www.bar.com_= 7 seconds', 48 '*RESULT a: a= [3,7] seconds\nAvg a: 5.000000seconds\n' + 49 'Sd a: 2.828427seconds'] 50 self.assertEquals(measurement_results.results, expected) 51 52 def test_basic_summary_nonuniform_results(self): 53 test_page_set = _MakePageSet() 54 55 measurement_results = SummarySavingPageMeasurementResults() 56 measurement_results.WillMeasurePage(test_page_set.pages[0]) 57 measurement_results.Add('a', 'seconds', 3) 58 measurement_results.Add('b', 'seconds', 10) 59 measurement_results.DidMeasurePage() 60 61 measurement_results.WillMeasurePage(test_page_set.pages[1]) 62 measurement_results.Add('a', 'seconds', 3) 63 measurement_results.Add('b', 'seconds', 10) 64 measurement_results.DidMeasurePage() 65 66 measurement_results.WillMeasurePage(test_page_set.pages[2]) 67 measurement_results.Add('a', 'seconds', 7) 68 # Note, page[2] does not report a 'b' metric. 69 measurement_results.DidMeasurePage() 70 71 measurement_results.PrintSummary() 72 expected = ['RESULT a_by_url: http___www.foo.com_= 3 seconds', 73 'RESULT a_by_url: http___www.bar.com_= 3 seconds', 74 'RESULT a_by_url: http___www.baz.com_= 7 seconds', 75 '*RESULT a: a= [3,3,7] seconds\nAvg a: 4.333333seconds\n' + 76 'Sd a: 2.309401seconds', 77 'RESULT b_by_url: http___www.foo.com_= 10 seconds', 78 'RESULT b_by_url: http___www.bar.com_= 10 seconds', 79 '*RESULT b: b= [10,10] seconds\nAvg b: 10.000000seconds'] 80 self.assertEquals(measurement_results.results, expected) 81 82 def test_basic_summary_pass_and_fail_page(self): 83 """If a page failed, only print summary for individual passing pages.""" 84 test_page_set = _MakePageSet() 85 86 measurement_results = SummarySavingPageMeasurementResults() 87 measurement_results.WillMeasurePage(test_page_set.pages[0]) 88 measurement_results.Add('a', 'seconds', 3) 89 measurement_results.DidMeasurePage() 90 measurement_results.AddFailureMessage(test_page_set.pages[0], 'message') 91 92 measurement_results.WillMeasurePage(test_page_set.pages[1]) 93 measurement_results.Add('a', 'seconds', 7) 94 measurement_results.DidMeasurePage() 95 96 measurement_results.PrintSummary() 97 expected = ['RESULT a_by_url: http___www.bar.com_= 7 seconds'] 98 self.assertEquals(measurement_results.results, expected) 99 100 def test_repeated_pageset_one_iteration_one_page_fails(self): 101 """Page fails on one iteration, no results for that page should print.""" 102 test_page_set = _MakePageSet() 103 104 measurement_results = SummarySavingPageMeasurementResults() 105 measurement_results.WillMeasurePage(test_page_set.pages[0]) 106 measurement_results.Add('a', 'seconds', 3) 107 measurement_results.DidMeasurePage() 108 109 measurement_results.WillMeasurePage(test_page_set.pages[1]) 110 measurement_results.Add('a', 'seconds', 7) 111 measurement_results.DidMeasurePage() 112 measurement_results.AddFailureMessage(test_page_set.pages[1], 'message') 113 114 measurement_results.WillMeasurePage(test_page_set.pages[0]) 115 measurement_results.Add('a', 'seconds', 4) 116 measurement_results.DidMeasurePage() 117 118 measurement_results.WillMeasurePage(test_page_set.pages[1]) 119 measurement_results.Add('a', 'seconds', 8) 120 measurement_results.DidMeasurePage() 121 122 measurement_results.PrintSummary() 123 expected = ['RESULT a_by_url: http___www.foo.com_= [3,4] seconds\n' + 124 'Avg a_by_url: 3.500000seconds\nSd a_by_url: 0.707107seconds'] 125 self.assertEquals(measurement_results.results, expected) 126 127 def test_repeated_pageset(self): 128 test_page_set = _MakePageSet() 129 130 measurement_results = SummarySavingPageMeasurementResults() 131 measurement_results.WillMeasurePage(test_page_set.pages[0]) 132 measurement_results.Add('a', 'seconds', 3) 133 measurement_results.DidMeasurePage() 134 135 measurement_results.WillMeasurePage(test_page_set.pages[1]) 136 measurement_results.Add('a', 'seconds', 7) 137 measurement_results.DidMeasurePage() 138 139 measurement_results.WillMeasurePage(test_page_set.pages[0]) 140 measurement_results.Add('a', 'seconds', 4) 141 measurement_results.DidMeasurePage() 142 143 measurement_results.WillMeasurePage(test_page_set.pages[1]) 144 measurement_results.Add('a', 'seconds', 8) 145 measurement_results.DidMeasurePage() 146 147 measurement_results.PrintSummary() 148 expected = ['RESULT a_by_url: http___www.foo.com_= [3,4] seconds\n' + 149 'Avg a_by_url: 3.500000seconds\nSd a_by_url: 0.707107seconds', 150 'RESULT a_by_url: http___www.bar.com_= [7,8] seconds\n' + 151 'Avg a_by_url: 7.500000seconds\nSd a_by_url: 0.707107seconds', 152 '*RESULT a: a= [3,7,4,8] seconds\n' + 153 'Avg a: 5.500000seconds\nSd a: 2.380476seconds' 154 ] 155 self.assertEquals( 156 measurement_results.results, 157 expected) 158 159 def test_repeated_pages(self): 160 test_page_set = _MakePageSet() 161 162 measurement_results = SummarySavingPageMeasurementResults() 163 measurement_results.WillMeasurePage(test_page_set.pages[0]) 164 measurement_results.Add('a', 'seconds', 3) 165 measurement_results.DidMeasurePage() 166 167 measurement_results.WillMeasurePage(test_page_set.pages[0]) 168 measurement_results.Add('a', 'seconds', 4) 169 measurement_results.DidMeasurePage() 170 171 measurement_results.WillMeasurePage(test_page_set.pages[1]) 172 measurement_results.Add('a', 'seconds', 7) 173 measurement_results.DidMeasurePage() 174 175 measurement_results.WillMeasurePage(test_page_set.pages[1]) 176 measurement_results.Add('a', 'seconds', 8) 177 measurement_results.DidMeasurePage() 178 179 measurement_results.PrintSummary() 180 expected = ['RESULT a_by_url: http___www.foo.com_= [3,4] seconds\n' + 181 'Avg a_by_url: 3.500000seconds\nSd a_by_url: 0.707107seconds', 182 'RESULT a_by_url: http___www.bar.com_= [7,8] seconds\n' + 183 'Avg a_by_url: 7.500000seconds\nSd a_by_url: 0.707107seconds', 184 '*RESULT a: a= [3,4,7,8] seconds\n' + 185 'Avg a: 5.500000seconds\nSd a: 2.380476seconds' 186 ] 187 self.assertEquals( 188 measurement_results.results, 189 expected) 190 191 def test_overall_results(self): 192 test_page_set = _MakePageSet() 193 194 measurement_results = SummarySavingPageMeasurementResults() 195 196 measurement_results.AddSummary('a', 'seconds', 1) 197 198 measurement_results.WillMeasurePage(test_page_set.pages[0]) 199 measurement_results.Add('b', 'seconds', 2) 200 measurement_results.DidMeasurePage() 201 202 measurement_results.WillMeasurePage(test_page_set.pages[1]) 203 measurement_results.Add('b', 'seconds', 3) 204 measurement_results.DidMeasurePage() 205 206 measurement_results.AddSummary('c', 'seconds', 4) 207 208 measurement_results.PrintSummary() 209 210 self.assertEquals( 211 measurement_results.results, 212 ['RESULT b_by_url: http___www.foo.com_= 2 seconds', 213 'RESULT b_by_url: http___www.bar.com_= 3 seconds', 214 '*RESULT b: b= [2,3] seconds\n' + 215 'Avg b: 2.500000seconds\nSd b: 0.707107seconds', 216 '*RESULT a: a= 1 seconds', 217 '*RESULT c: c= 4 seconds']) 218 219 def test_overall_results_trace_tag(self): 220 test_page_set = _MakePageSet() 221 222 measurement_results = SummarySavingPageMeasurementResults(trace_tag='_ref') 223 224 measurement_results.AddSummary('a', 'seconds', 1) 225 226 measurement_results.WillMeasurePage(test_page_set.pages[0]) 227 measurement_results.Add('b', 'seconds', 2) 228 measurement_results.DidMeasurePage() 229 230 measurement_results.WillMeasurePage(test_page_set.pages[1]) 231 measurement_results.Add('b', 'seconds', 3) 232 measurement_results.DidMeasurePage() 233 234 measurement_results.AddSummary('c', 'seconds', 4) 235 236 measurement_results.PrintSummary() 237 238 self.assertEquals( 239 measurement_results.results, 240 ['*RESULT b: b_ref= [2,3] seconds\n' + 241 'Avg b: 2.500000seconds\nSd b: 0.707107seconds', 242 '*RESULT a: a_ref= 1 seconds', 243 '*RESULT c: c_ref= 4 seconds']) 244 245 def test_histogram(self): 246 test_page_set = _MakePageSet() 247 248 measurement_results = SummarySavingPageMeasurementResults() 249 measurement_results.WillMeasurePage(test_page_set.pages[0]) 250 measurement_results.Add('a', '', 251 '{"buckets": [{"low": 1, "high": 2, "count": 1}]}', 252 data_type='histogram') 253 measurement_results.DidMeasurePage() 254 255 measurement_results.WillMeasurePage(test_page_set.pages[1]) 256 measurement_results.Add('a', '', 257 '{"buckets": [{"low": 2, "high": 3, "count": 1}]}', 258 data_type='histogram') 259 measurement_results.DidMeasurePage() 260 261 measurement_results.PrintSummary() 262 263 expected = [ 264 'HISTOGRAM a_by_url: http___www.foo.com_= ' + 265 '{"buckets": [{"low": 1, "high": 2, "count": 1}]}\n' + 266 'Avg a_by_url: 1.500000', 267 'HISTOGRAM a_by_url: http___www.bar.com_= ' + 268 '{"buckets": [{"low": 2, "high": 3, "count": 1}]}\n' + 269 'Avg a_by_url: 2.500000'] 270 self.assertEquals(measurement_results.results, expected) 271