1# Copyright 2014 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import math 6import os 7import unittest 8 9from telemetry import story 10from telemetry.internal.results import page_test_results 11from telemetry import page as page_module 12from telemetry.value import failure 13from telemetry.value import histogram 14from telemetry.value import improvement_direction 15from telemetry.value import list_of_scalar_values 16from telemetry.value import scalar 17from telemetry.value import summary as summary_module 18 19 20class TestBase(unittest.TestCase): 21 def setUp(self): 22 story_set = story.StorySet(base_dir=os.path.dirname(__file__)) 23 story_set.AddStory( 24 page_module.Page('http://www.bar.com/', story_set, story_set.base_dir)) 25 story_set.AddStory( 26 page_module.Page('http://www.baz.com/', story_set, story_set.base_dir)) 27 story_set.AddStory( 28 page_module.Page('http://www.foo.com/', story_set, story_set.base_dir)) 29 self.story_set = story_set 30 31 @property 32 def pages(self): 33 return self.story_set.stories 34 35 36class SummaryTest(TestBase): 37 def testBasicSummary(self): 38 page0 = self.pages[0] 39 page1 = self.pages[1] 40 41 results = page_test_results.PageTestResults() 42 43 results.WillRunPage(page0) 44 v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3, 45 improvement_direction=improvement_direction.UP) 46 results.AddValue(v0) 47 results.DidRunPage(page0) 48 49 results.WillRunPage(page1) 50 v1 = scalar.ScalarValue(page1, 'a', 'seconds', 7, 51 improvement_direction=improvement_direction.UP) 52 results.AddValue(v1) 53 results.DidRunPage(page1) 54 55 summary = summary_module.Summary(results.all_page_specific_values) 56 values = summary.interleaved_computed_per_page_values_and_summaries 57 58 v0_list = list_of_scalar_values.ListOfScalarValues( 59 page0, 'a', 'seconds', [3], 60 improvement_direction=improvement_direction.UP) 61 v1_list = list_of_scalar_values.ListOfScalarValues( 62 page1, 'a', 'seconds', [7], 63 improvement_direction=improvement_direction.UP) 64 # Std is 0 because we only have one measurement per page. 65 merged_value = list_of_scalar_values.ListOfScalarValues( 66 None, 'a', 'seconds', [3, 7], std=0.0, 67 improvement_direction=improvement_direction.UP) 68 69 self.assertEquals(3, len(values)) 70 self.assertIn(v0_list, values) 71 self.assertIn(v1_list, values) 72 self.assertIn(merged_value, values) 73 74 def testBasicSummaryWithOnlyOnePage(self): 75 page0 = self.pages[0] 76 77 results = page_test_results.PageTestResults() 78 79 results.WillRunPage(page0) 80 v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3, 81 improvement_direction=improvement_direction.UP) 82 results.AddValue(v0) 83 results.DidRunPage(page0) 84 85 summary = summary_module.Summary(results.all_page_specific_values) 86 values = summary.interleaved_computed_per_page_values_and_summaries 87 88 v0_list = list_of_scalar_values.ListOfScalarValues( 89 page0, 'a', 'seconds', [3], 90 improvement_direction=improvement_direction.UP) 91 merged_list = list_of_scalar_values.ListOfScalarValues( 92 None, 'a', 'seconds', [3], 93 improvement_direction=improvement_direction.UP) 94 95 self.assertEquals(2, len(values)) 96 self.assertIn(v0_list, values) 97 self.assertIn(merged_list, values) 98 99 def testBasicSummaryNonuniformResults(self): 100 page0 = self.pages[0] 101 page1 = self.pages[1] 102 page2 = self.pages[2] 103 104 results = page_test_results.PageTestResults() 105 results.WillRunPage(page0) 106 v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3, 107 improvement_direction=improvement_direction.UP) 108 results.AddValue(v0) 109 v1 = scalar.ScalarValue(page0, 'b', 'seconds', 10, 110 improvement_direction=improvement_direction.UP) 111 results.AddValue(v1) 112 results.DidRunPage(page0) 113 114 results.WillRunPage(page1) 115 v2 = scalar.ScalarValue(page1, 'a', 'seconds', 3, 116 improvement_direction=improvement_direction.UP) 117 results.AddValue(v2) 118 v3 = scalar.ScalarValue(page1, 'b', 'seconds', 10, 119 improvement_direction=improvement_direction.UP) 120 results.AddValue(v3) 121 results.DidRunPage(page1) 122 123 results.WillRunPage(page2) 124 v4 = scalar.ScalarValue(page2, 'a', 'seconds', 7, 125 improvement_direction=improvement_direction.UP) 126 results.AddValue(v4) 127 # Note, page[2] does not report a 'b' metric. 128 results.DidRunPage(page2) 129 130 summary = summary_module.Summary(results.all_page_specific_values) 131 values = summary.interleaved_computed_per_page_values_and_summaries 132 133 v0_list = list_of_scalar_values.ListOfScalarValues( 134 page0, 'a', 'seconds', [3], 135 improvement_direction=improvement_direction.UP) 136 v1_list = list_of_scalar_values.ListOfScalarValues( 137 page0, 'b', 'seconds', [10], 138 improvement_direction=improvement_direction.UP) 139 v2_list = list_of_scalar_values.ListOfScalarValues( 140 page1, 'a', 'seconds', [3], 141 improvement_direction=improvement_direction.UP) 142 v3_list = list_of_scalar_values.ListOfScalarValues( 143 page1, 'b', 'seconds', [10], 144 improvement_direction=improvement_direction.UP) 145 v4_list = list_of_scalar_values.ListOfScalarValues( 146 page2, 'a', 'seconds', [7], 147 improvement_direction=improvement_direction.UP) 148 # Std is 0 because we only have one measurement per page. 149 a_summary = list_of_scalar_values.ListOfScalarValues( 150 None, 'a', 'seconds', [3, 3, 7], std=0.0, 151 improvement_direction=improvement_direction.UP) 152 b_summary = list_of_scalar_values.ListOfScalarValues( 153 None, 'b', 'seconds', [10, 10], std=0.0, 154 improvement_direction=improvement_direction.UP) 155 156 self.assertEquals(7, len(values)) 157 self.assertIn(v0_list, values) 158 self.assertIn(v1_list, values) 159 self.assertIn(v2_list, values) 160 self.assertIn(v3_list, values) 161 self.assertIn(v4_list, values) 162 self.assertIn(a_summary, values) 163 self.assertIn(b_summary, values) 164 165 def testBasicSummaryPassAndFailPage(self): 166 """If a page failed, only print summary for individual pages.""" 167 page0 = self.pages[0] 168 page1 = self.pages[1] 169 170 results = page_test_results.PageTestResults() 171 results.WillRunPage(page0) 172 v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3, 173 improvement_direction=improvement_direction.UP) 174 results.AddValue(v0) 175 v1 = failure.FailureValue.FromMessage(page0, 'message') 176 results.AddValue(v1) 177 results.DidRunPage(page0) 178 179 results.WillRunPage(page1) 180 v2 = scalar.ScalarValue(page1, 'a', 'seconds', 7, 181 improvement_direction=improvement_direction.UP) 182 results.AddValue(v2) 183 results.DidRunPage(page1) 184 185 summary = summary_module.Summary(results.all_page_specific_values) 186 values = summary.interleaved_computed_per_page_values_and_summaries 187 188 v0_list = list_of_scalar_values.ListOfScalarValues( 189 page0, 'a', 'seconds', [3], 190 improvement_direction=improvement_direction.UP) 191 v2_list = list_of_scalar_values.ListOfScalarValues( 192 page1, 'a', 'seconds', [7], 193 improvement_direction=improvement_direction.UP) 194 195 self.assertEquals(2, len(values)) 196 self.assertIn(v0_list, values) 197 self.assertIn(v2_list, values) 198 199 def testRepeatedPagesetOneIterationOnePageFails(self): 200 """Page fails on one iteration, no averaged results should print.""" 201 page0 = self.pages[0] 202 page1 = self.pages[1] 203 204 results = page_test_results.PageTestResults() 205 results.WillRunPage(page0) 206 v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3, 207 improvement_direction=improvement_direction.UP) 208 results.AddValue(v0) 209 results.DidRunPage(page0) 210 211 results.WillRunPage(page1) 212 v1 = scalar.ScalarValue(page1, 'a', 'seconds', 7, 213 improvement_direction=improvement_direction.UP) 214 results.AddValue(v1) 215 v2 = failure.FailureValue.FromMessage(page1, 'message') 216 results.AddValue(v2) 217 results.DidRunPage(page1) 218 219 results.WillRunPage(page0) 220 v3 = scalar.ScalarValue(page0, 'a', 'seconds', 4, 221 improvement_direction=improvement_direction.UP) 222 results.AddValue(v3) 223 results.DidRunPage(page0) 224 225 results.WillRunPage(page1) 226 v4 = scalar.ScalarValue(page1, 'a', 'seconds', 8, 227 improvement_direction=improvement_direction.UP) 228 results.AddValue(v4) 229 results.DidRunPage(page1) 230 231 summary = summary_module.Summary(results.all_page_specific_values) 232 values = summary.interleaved_computed_per_page_values_and_summaries 233 234 page0_aggregated = list_of_scalar_values.ListOfScalarValues( 235 page0, 'a', 'seconds', [3, 4], 236 improvement_direction=improvement_direction.UP) 237 page1_aggregated = list_of_scalar_values.ListOfScalarValues( 238 page1, 'a', 'seconds', [7, 8], 239 improvement_direction=improvement_direction.UP) 240 241 self.assertEquals(2, len(values)) 242 self.assertIn(page0_aggregated, values) 243 self.assertIn(page1_aggregated, values) 244 245 def testRepeatedPages(self): 246 page0 = self.pages[0] 247 page1 = self.pages[1] 248 249 results = page_test_results.PageTestResults() 250 results.WillRunPage(page0) 251 v0 = scalar.ScalarValue(page0, 'a', 'seconds', 3, 252 improvement_direction=improvement_direction.UP) 253 results.AddValue(v0) 254 results.DidRunPage(page0) 255 256 results.WillRunPage(page0) 257 v2 = scalar.ScalarValue(page0, 'a', 'seconds', 4, 258 improvement_direction=improvement_direction.UP) 259 results.AddValue(v2) 260 results.DidRunPage(page0) 261 262 results.WillRunPage(page1) 263 v1 = scalar.ScalarValue(page1, 'a', 'seconds', 7, 264 improvement_direction=improvement_direction.UP) 265 results.AddValue(v1) 266 results.DidRunPage(page1) 267 268 results.WillRunPage(page1) 269 v3 = scalar.ScalarValue(page1, 'a', 'seconds', 8, 270 improvement_direction=improvement_direction.UP) 271 results.AddValue(v3) 272 results.DidRunPage(page1) 273 274 summary = summary_module.Summary(results.all_page_specific_values) 275 values = summary.interleaved_computed_per_page_values_and_summaries 276 277 page0_aggregated = list_of_scalar_values.ListOfScalarValues( 278 page0, 'a', 'seconds', [3, 4], 279 improvement_direction=improvement_direction.UP) 280 page1_aggregated = list_of_scalar_values.ListOfScalarValues( 281 page1, 'a', 'seconds', [7, 8], 282 improvement_direction=improvement_direction.UP) 283 # Std is computed using pooled standard deviation. 284 a_summary = list_of_scalar_values.ListOfScalarValues( 285 None, 'a', 'seconds', [3, 4, 7, 8], std=math.sqrt(0.5), 286 improvement_direction=improvement_direction.UP) 287 288 self.assertEquals(3, len(values)) 289 self.assertIn(page0_aggregated, values) 290 self.assertIn(page1_aggregated, values) 291 self.assertIn(a_summary, values) 292 293 def testPageRunsTwice(self): 294 page0 = self.pages[0] 295 296 results = page_test_results.PageTestResults() 297 298 results.WillRunPage(page0) 299 v0 = scalar.ScalarValue(page0, 'b', 'seconds', 2, 300 improvement_direction=improvement_direction.UP) 301 results.AddValue(v0) 302 results.DidRunPage(page0) 303 304 results.WillRunPage(page0) 305 v1 = scalar.ScalarValue(page0, 'b', 'seconds', 3, 306 improvement_direction=improvement_direction.UP) 307 results.AddValue(v1) 308 results.DidRunPage(page0) 309 310 summary = summary_module.Summary(results.all_page_specific_values) 311 values = summary.interleaved_computed_per_page_values_and_summaries 312 313 page0_aggregated = list_of_scalar_values.ListOfScalarValues( 314 page0, 'b', 'seconds', [2, 3], 315 improvement_direction=improvement_direction.UP) 316 b_summary = list_of_scalar_values.ListOfScalarValues( 317 None, 'b', 'seconds', [2, 3], 318 improvement_direction=improvement_direction.UP) 319 320 self.assertEquals(2, len(values)) 321 self.assertIn(page0_aggregated, values) 322 self.assertIn(b_summary, values) 323 324 def testListValue(self): 325 page0 = self.pages[0] 326 page1 = self.pages[1] 327 328 results = page_test_results.PageTestResults() 329 330 results.WillRunPage(page0) 331 v0 = list_of_scalar_values.ListOfScalarValues( 332 page0, 'b', 'seconds', [2, 2], 333 improvement_direction=improvement_direction.UP) 334 results.AddValue(v0) 335 results.DidRunPage(page0) 336 337 results.WillRunPage(page1) 338 v1 = list_of_scalar_values.ListOfScalarValues( 339 page1, 'b', 'seconds', [3, 3], 340 improvement_direction=improvement_direction.UP) 341 results.AddValue(v1) 342 results.DidRunPage(page1) 343 344 summary = summary_module.Summary(results.all_page_specific_values) 345 values = summary.interleaved_computed_per_page_values_and_summaries 346 347 b_summary = list_of_scalar_values.ListOfScalarValues( 348 None, 'b', 'seconds', [2, 2, 3, 3], std=0.0, 349 improvement_direction=improvement_direction.UP) 350 351 self.assertEquals(3, len(values)) 352 self.assertIn(v0, values) 353 self.assertIn(v1, values) 354 self.assertIn(b_summary, values) 355 356 def testHistogram(self): 357 page0 = self.pages[0] 358 page1 = self.pages[1] 359 360 results = page_test_results.PageTestResults() 361 results.WillRunPage(page0) 362 v0 = histogram.HistogramValue( 363 page0, 'a', 'units', 364 raw_value_json='{"buckets": [{"low": 1, "high": 2, "count": 1}]}', 365 important=False, improvement_direction=improvement_direction.UP) 366 results.AddValue(v0) 367 results.DidRunPage(page0) 368 369 results.WillRunPage(page1) 370 v1 = histogram.HistogramValue( 371 page1, 'a', 'units', 372 raw_value_json='{"buckets": [{"low": 2, "high": 3, "count": 1}]}', 373 important=False, improvement_direction=improvement_direction.UP) 374 results.AddValue(v1) 375 results.DidRunPage(page1) 376 377 summary = summary_module.Summary(results.all_page_specific_values) 378 values = summary.interleaved_computed_per_page_values_and_summaries 379 380 self.assertEquals(2, len(values)) 381 self.assertIn(v0, values) 382 self.assertIn(v1, values) 383 384 def testSummaryUsesKeyFunc(self): 385 page0 = self.pages[0] 386 page1 = self.pages[1] 387 388 results = page_test_results.PageTestResults() 389 390 results.WillRunPage(page0) 391 v0 = scalar.ScalarValue(page0, 'a', 'seconds', 20, 392 improvement_direction=improvement_direction.UP) 393 results.AddValue(v0) 394 395 v1 = scalar.ScalarValue(page0, 'b', 'seconds', 42, 396 improvement_direction=improvement_direction.UP) 397 results.AddValue(v1) 398 results.DidRunPage(page0) 399 400 results.WillRunPage(page1) 401 v2 = scalar.ScalarValue(page1, 'a', 'seconds', 20, 402 improvement_direction=improvement_direction.UP) 403 results.AddValue(v2) 404 405 v3 = scalar.ScalarValue(page1, 'b', 'seconds', 42, 406 improvement_direction=improvement_direction.UP) 407 results.AddValue(v3) 408 results.DidRunPage(page1) 409 410 summary = summary_module.Summary( 411 results.all_page_specific_values, 412 key_func=lambda v: True) 413 values = summary.interleaved_computed_per_page_values_and_summaries 414 415 v0_list = list_of_scalar_values.ListOfScalarValues( 416 page0, 'a', 'seconds', [20, 42], 417 improvement_direction=improvement_direction.UP) 418 v2_list = list_of_scalar_values.ListOfScalarValues( 419 page1, 'a', 'seconds', [20, 42], 420 improvement_direction=improvement_direction.UP) 421 # Std is computed using pooled standard deviation. 422 merged_value = list_of_scalar_values.ListOfScalarValues( 423 None, 'a', 'seconds', [20, 42, 20, 42], std=math.sqrt(242.0), 424 improvement_direction=improvement_direction.UP) 425 426 self.assertEquals(3, len(values)) 427 self.assertIn(v0_list, values) 428 self.assertIn(v2_list, values) 429 self.assertIn(merged_value, values) 430