1#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Verify perf_expectations.json can be loaded using simplejson.
7
8perf_expectations.json is a JSON-formatted file.  This script verifies
9that simplejson can load it correctly.  It should catch most common
10formatting problems.
11"""
12
13import subprocess
14import sys
15import os
16import unittest
17import re
18
19simplejson = None
20
21def OnTestsLoad():
22  old_path = sys.path
23  script_path = os.path.dirname(sys.argv[0])
24  load_path = None
25  global simplejson
26
27  # This test script should be stored in src/tools/perf_expectations/.  That
28  # directory will most commonly live in 2 locations:
29  #
30  #   - a regular Chromium checkout, in which case src/third_party
31  #     is where to look for simplejson
32  #
33  #   - a buildbot checkout, in which case .../pylibs is where
34  #     to look for simplejson
35  #
36  # Locate and install the correct path based on what we can find.
37  #
38  for path in ('../../../third_party', '../../../../../pylibs'):
39    path = os.path.join(script_path, path)
40    if os.path.exists(path) and os.path.isdir(path):
41      load_path = os.path.abspath(path)
42      break
43
44  if load_path is None:
45    msg = "%s expects to live within a Chromium checkout" % sys.argv[0]
46    raise Exception, "Error locating simplejson load path (%s)" % msg
47
48  # Try importing simplejson once.  If this succeeds, we found it and will
49  # load it again later properly.  Fail if we cannot load it.
50  sys.path.append(load_path)
51  try:
52    import simplejson as Simplejson
53    simplejson = Simplejson
54  except ImportError, e:
55    msg = "%s expects to live within a Chromium checkout" % sys.argv[0]
56    raise Exception, "Error trying to import simplejson from %s (%s)" % \
57                     (load_path, msg)
58  finally:
59    sys.path = old_path
60  return True
61
62def LoadJsonFile(filename):
63  f = open(filename, 'r')
64  try:
65    data = simplejson.load(f)
66  except ValueError, e:
67    f.seek(0)
68    print "Error reading %s:\n%s" % (filename,
69                                     f.read()[:50]+'...')
70    raise e
71  f.close()
72  return data
73
74OnTestsLoad()
75
76CONFIG_JSON = os.path.join(os.path.dirname(sys.argv[0]),
77                           '../chromium_perf_expectations.cfg')
78MAKE_EXPECTATIONS = os.path.join(os.path.dirname(sys.argv[0]),
79                                 '../make_expectations.py')
80PERF_EXPECTATIONS = os.path.join(os.path.dirname(sys.argv[0]),
81                                 '../perf_expectations.json')
82
83
84class PerfExpectationsUnittest(unittest.TestCase):
85  def testPerfExpectations(self):
86    # Test data is dictionary.
87    perf_data = LoadJsonFile(PERF_EXPECTATIONS)
88    if not isinstance(perf_data, dict):
89      raise Exception('perf expectations is not a dict')
90
91    # Test the 'load' key.
92    if not 'load' in perf_data:
93      raise Exception("perf expectations is missing a load key")
94    if not isinstance(perf_data['load'], bool):
95      raise Exception("perf expectations load key has non-bool value")
96
97    # Test all key values are dictionaries.
98    bad_keys = []
99    for key in perf_data:
100      if key == 'load':
101        continue
102      if not isinstance(perf_data[key], dict):
103        bad_keys.append(key)
104    if len(bad_keys) > 0:
105      msg = "perf expectations keys have non-dict values"
106      raise Exception("%s: %s" % (msg, bad_keys))
107
108    # Test all key values have delta and var keys.
109    for key in perf_data:
110      if key == 'load':
111        continue
112
113      # First check if regress/improve is in the key's data.
114      if 'regress' in perf_data[key]:
115        if 'improve' not in perf_data[key]:
116          bad_keys.append(key)
117        if (not isinstance(perf_data[key]['regress'], int) and
118            not isinstance(perf_data[key]['regress'], float)):
119          bad_keys.append(key)
120        if (not isinstance(perf_data[key]['improve'], int) and
121            not isinstance(perf_data[key]['improve'], float)):
122          bad_keys.append(key)
123      else:
124        # Otherwise check if delta/var is in the key's data.
125        if 'delta' not in perf_data[key] or 'var' not in perf_data[key]:
126          bad_keys.append(key)
127        if (not isinstance(perf_data[key]['delta'], int) and
128            not isinstance(perf_data[key]['delta'], float)):
129          bad_keys.append(key)
130        if (not isinstance(perf_data[key]['var'], int) and
131            not isinstance(perf_data[key]['var'], float)):
132          bad_keys.append(key)
133
134    if len(bad_keys) > 0:
135      msg = "perf expectations key values missing or invalid delta/var"
136      raise Exception("%s: %s" % (msg, bad_keys))
137
138    # Test all keys have the correct format.
139    for key in perf_data:
140      if key == 'load':
141        continue
142      # tools/buildbot/scripts/master/log_parser.py should have a matching
143      # regular expression.
144      if not re.match(r"^([\w\.-]+)/([\w\.-]+)/([\w\.-]+)/([\w\.-]+)$", key):
145        bad_keys.append(key)
146    if len(bad_keys) > 0:
147      msg = "perf expectations keys in bad format, expected a/b/c/d"
148      raise Exception("%s: %s" % (msg, bad_keys))
149
150  def testNoUpdatesNeeded(self):
151    p = subprocess.Popen([MAKE_EXPECTATIONS, '-s'], stdout=subprocess.PIPE)
152    p.wait();
153    self.assertEqual(p.returncode, 0,
154        msg='Update expectations first by running ./make_expectations.py')
155
156  def testConfigFile(self):
157    # Test that the config file can be parsed as JSON.
158    config = LoadJsonFile(CONFIG_JSON)
159    # Require the following keys.
160    if 'base_url' not in config:
161      raise Exception('base_url not specified in config file')
162    if 'perf_file' not in config:
163      raise Exception('perf_file not specified in config file')
164
165
166if __name__ == '__main__':
167  unittest.main()
168