136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# Copyright 2012 the V8 project authors. All rights reserved. 236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# Redistribution and use in source and binary forms, with or without 336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# modification, are permitted provided that the following conditions are 436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# met: 536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# 636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# * Redistributions of source code must retain the above copyright 736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# notice, this list of conditions and the following disclaimer. 836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# * Redistributions in binary form must reproduce the above 936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# copyright notice, this list of conditions and the following 1036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# disclaimer in the documentation and/or other materials provided 1136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# with the distribution. 1236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# * Neither the name of Google Inc. nor the names of its 1336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# contributors may be used to endorse or promote products derived 1436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# from this software without specific prior written permission. 1536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# 1636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 1736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 2036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 2136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 2236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 2636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 28dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hines 2936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesimport sys 30dce4a407a24b04eebc6a376f8e62b41aaa7b071fStephen Hinesimport time 3136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesfrom . import statusfile 3336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 3536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen HinesREPORT_TEMPLATE = ( 3636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines"""Total: %(total)i tests 3736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines * %(skipped)4d tests will be skipped 3836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines * %(timeout)4d tests are expected to timeout sometimes 3936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines * %(nocrash)4d tests are expected to be flaky but not crash 4036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines * %(pass)4d tests are expected to pass 4136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines * %(fail_ok)4d tests are expected to fail that we won't fix 4236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines * %(fail)4d tests are expected to fail that we should fix""") 4336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 4436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 4536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesdef PrintReport(tests): 4636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines total = len(tests) 4736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines skipped = timeout = nocrash = passes = fail_ok = fail = 0 4836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for t in tests: 4936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if "outcomes" not in dir(t) or not t.outcomes: 5036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines passes += 1 5136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue 5236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines o = t.outcomes 5336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if statusfile.DoSkip(o): 5436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines skipped += 1 5536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines continue 5636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if statusfile.TIMEOUT in o: timeout += 1 5736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if statusfile.IsPassOrFail(o): nocrash += 1 5836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if list(o) == [statusfile.PASS]: passes += 1 5936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if statusfile.IsFailOk(o): fail_ok += 1 6036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if list(o) == [statusfile.FAIL]: fail += 1 6136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines print REPORT_TEMPLATE % { 6236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "total": total, 6336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "skipped": skipped, 6436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "timeout": timeout, 6536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "nocrash": nocrash, 6636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "pass": passes, 6736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "fail_ok": fail_ok, 6836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines "fail": fail 6936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines } 7036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 7136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 7236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesdef PrintTestSource(tests): 7336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for test in tests: 7436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines suite = test.suite 7536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines source = suite.GetSourceForTest(test).strip() 7636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if len(source) > 0: 7736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines print "--- begin source: %s/%s ---" % (suite.name, test.path) 7836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines print source 7936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines print "--- end source: %s/%s ---" % (suite.name, test.path) 8036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 8136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 8236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesdef FormatTime(d): 8336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines millis = round(d * 1000) % 1000 8436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return time.strftime("%M:%S.", time.gmtime(d)) + ("%03i" % millis) 8536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 8636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 8736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hinesdef PrintTestDurations(suites, overall_time): 8836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines # Write the times to stderr to make it easy to separate from the 8936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines # test output. 9036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines print 9136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines sys.stderr.write("--- Total time: %s ---\n" % FormatTime(overall_time)) 9236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines timed_tests = [ t for s in suites for t in s.tests 9336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if t.duration is not None ] 9436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines timed_tests.sort(lambda a, b: cmp(b.duration, a.duration)) 9536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines index = 1 9636b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines for entry in timed_tests[:20]: 9736b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines t = FormatTime(entry.duration) 9836b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines sys.stderr.write("%4i (%s) %s\n" % (index, t, entry.GetLabel())) 9936b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines index += 1 10036b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines