html_output_formatter_unittest.py revision cef7893435aa41160dd1255c43cb8498279738cc 
1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4import datetime
5import os
6import StringIO
7import unittest
8
9from telemetry import benchmark
10from telemetry import story
11from telemetry.internal.results import html_output_formatter
12from telemetry.internal.results import page_test_results
13from telemetry import page as page_module
14from telemetry.value import improvement_direction
15from telemetry.value import scalar
16
17
18def _MakeStorySet():
19  story_set = story.StorySet(base_dir=os.path.dirname(__file__))
20  story_set.AddStory(
21      page_module.Page('http://www.foo.com/', story_set, story_set.base_dir))
22  story_set.AddStory(
23      page_module.Page('http://www.bar.com/', story_set, story_set.base_dir))
24  story_set.AddStory(
25      page_module.Page('http://www.baz.com/', story_set, story_set.base_dir))
26  return story_set
27
28
29class DeterministicHtmlOutputFormatter(
30    html_output_formatter.HtmlOutputFormatter):
31  def _GetBuildTime(self):
32    return datetime.datetime(1998, 9, 4, 13, 0, 0, 7777)
33
34  def _GetRevision(self):
35    return 'revision'
36
37class FakeMetadataForTest(benchmark.BenchmarkMetadata):
38  def __init__(self):
39    super(FakeMetadataForTest, self).__init__('test_name')
40
41# Wrap string IO with a .name property so that it behaves more like a file.
42class StringIOFile(StringIO.StringIO):
43  name = 'fake_output_file'
44
45
46class HtmlOutputFormatterTest(unittest.TestCase):
47
48  def setUp(self):
49    self.maxDiff = 100000
50
51  def test_basic_summary(self):
52    test_story_set = _MakeStorySet()
53    output_file = StringIOFile()
54
55    # Run the first time and verify the results are written to the HTML file.
56    results = page_test_results.PageTestResults()
57    results.WillRunPage(test_story_set.stories[0])
58    results.AddValue(scalar.ScalarValue(
59        test_story_set.stories[0], 'a', 'seconds', 3,
60        improvement_direction=improvement_direction.DOWN))
61    results.DidRunPage(test_story_set.stories[0])
62
63    results.WillRunPage(test_story_set.stories[1])
64    results.AddValue(scalar.ScalarValue(
65        test_story_set.stories[1], 'a', 'seconds', 7,
66        improvement_direction=improvement_direction.DOWN))
67    results.DidRunPage(test_story_set.stories[1])
68
69    formatter = DeterministicHtmlOutputFormatter(
70        output_file, FakeMetadataForTest(), False, False, 'browser_type')
71    formatter.Format(results)
72    expected = {
73      "platform": "browser_type",
74      "buildTime": "1998-09-04T13:00:00.007777",
75      "label": 'test_name (1998-09-04 13:00:00)',
76      "tests": {
77        "test_name": {
78          "metrics": {
79            "a": {
80              "current": [3, 7],
81              "std": 2.8284271247461903,
82              "units": "seconds",
83              "important": True
84            },
85            "telemetry_page_measurement_results.num_failed": {
86              "current": [0],
87              "units": "count",
88              "important": False
89            },
90            "a.http://www.bar.com/": {
91              "current": [7],
92              "std": 0.0,
93              "units": "seconds",
94              "important": False
95            },
96            "a.http://www.foo.com/": {
97              "current": [3],
98              "std": 0.0,
99              "units": "seconds",
100              "important": False
101            }
102          }
103        }
104      },
105    }
106    self.assertEquals(expected, formatter.GetResults())
107
108    # Run the second time and verify the results are appended to the HTML file.
109    output_file.seek(0)
110    results = page_test_results.PageTestResults()
111    results.WillRunPage(test_story_set.stories[0])
112    results.AddValue(scalar.ScalarValue(
113        test_story_set.stories[0], 'a', 'seconds', 4,
114        improvement_direction=improvement_direction.DOWN))
115    results.DidRunPage(test_story_set.stories[0])
116
117    results.WillRunPage(test_story_set.stories[1])
118    results.AddValue(scalar.ScalarValue(
119        test_story_set.stories[1], 'a', 'seconds', 8,
120        improvement_direction=improvement_direction.DOWN))
121    results.DidRunPage(test_story_set.stories[1])
122
123    formatter = DeterministicHtmlOutputFormatter(
124        output_file, FakeMetadataForTest(), False, False, 'browser_type')
125    formatter.Format(results)
126    expected = [
127      {
128        "platform": "browser_type",
129        "buildTime": "1998-09-04T13:00:00.007777",
130        "label": 'test_name (1998-09-04 13:00:00)',
131        "tests": {
132          "test_name": {
133            "metrics": {
134              "a": {
135                "current": [3, 7],
136                "units": "seconds",
137                "std": 2.8284271247461903,
138                "important": True
139              },
140              "telemetry_page_measurement_results.num_failed": {
141                "current": [0],
142                "units": "count",
143                "important": False
144              },
145              "a.http://www.bar.com/": {
146                "current": [7],
147                "std": 0.0,
148                "units": "seconds",
149                "important": False
150              },
151              "a.http://www.foo.com/": {
152                "current": [3],
153                "std": 0.0,
154                "units": "seconds",
155                "important": False
156              }
157            }
158          }
159        },
160      },
161      {
162        "platform": "browser_type",
163        "buildTime": "1998-09-04T13:00:00.007777",
164        "label": 'test_name (1998-09-04 13:00:00)',
165        "tests": {
166          "test_name": {
167            "metrics": {
168              "a": {
169                "current": [4, 8],
170                'std': 2.8284271247461903,
171                "units": "seconds",
172                "important": True
173              },
174              "telemetry_page_measurement_results.num_failed": {
175                "current": [0],
176                "units": "count",
177                "important": False,
178              },
179              "a.http://www.bar.com/": {
180                "current": [8],
181                "std": 0.0,
182                "units": "seconds",
183                "important": False
184              },
185              "a.http://www.foo.com/": {
186                "current": [4],
187                "std": 0.0,
188                "units": "seconds",
189                "important": False
190              }
191            }
192          }
193        },
194      }]
195    self.assertEquals(expected, formatter.GetCombinedResults())
196    last_output_len = len(output_file.getvalue())
197
198    # Now reset the results and verify the old ones are gone.
199    output_file.seek(0)
200    results = page_test_results.PageTestResults()
201    results.WillRunPage(test_story_set.stories[0])
202    results.AddValue(scalar.ScalarValue(
203        test_story_set.stories[0], 'a', 'seconds', 5,
204        improvement_direction=improvement_direction.DOWN))
205    results.DidRunPage(test_story_set.stories[0])
206
207    results.WillRunPage(test_story_set.stories[1])
208    results.AddValue(scalar.ScalarValue(
209        test_story_set.stories[1], 'a', 'seconds', 9,
210        improvement_direction=improvement_direction.DOWN))
211    results.AddValue(scalar.ScalarValue(
212        test_story_set.stories[1], 'b', 'seconds', 20, tir_label='foo'))
213    results.DidRunPage(test_story_set.stories[1])
214
215    formatter = DeterministicHtmlOutputFormatter(
216       output_file, FakeMetadataForTest(), True, False, 'browser_type')
217    formatter.Format(results)
218    expected = [{
219      "platform": "browser_type",
220      "buildTime": "1998-09-04T13:00:00.007777",
221      "label": 'test_name (1998-09-04 13:00:00)',
222      "tests": {
223        "test_name": {
224          "metrics": {
225            "a": {
226              "current": [5, 9],
227              'std': 2.8284271247461903,
228              "units": "seconds",
229              "important": True
230            },
231            "telemetry_page_measurement_results.num_failed": {
232              "current": [0],
233              "units": "count",
234              "important": False
235            },
236            "a.http://www.bar.com/": {
237              "current": [9],
238              "std": 0.0,
239              "units": "seconds",
240              "important": False
241            },
242            "a.http://www.foo.com/": {
243              "current": [5],
244              "std": 0.0,
245              "units": "seconds",
246              "important": False
247            },
248            "foo-b.http://www.bar.com/": {
249              "current": [20],
250              "std": 0.0,
251              "units": "seconds",
252              "important": False
253            },
254            "foo-b": {
255              "current": [20],
256              "std": 0.0,
257              "units": "seconds",
258              "important": True
259            }
260          }
261        }
262      },
263    }]
264    self.assertEquals(expected, formatter.GetCombinedResults())
265    self.assertTrue(len(output_file.getvalue()) < last_output_len)
266