1be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbarimport os 2be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar 3ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar# Test result codes. 4be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar 5ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbarclass ResultCode(object): 6ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbar """Test result codes.""" 7ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbar 88253cc047de523fd2097e4d85417316ff9a37ad5Daniel Dunbar # We override __new__ and __getnewargs__ to ensure that pickling still 98253cc047de523fd2097e4d85417316ff9a37ad5Daniel Dunbar # provides unique ResultCode objects in any particular instance. 108253cc047de523fd2097e4d85417316ff9a37ad5Daniel Dunbar _instances = {} 118253cc047de523fd2097e4d85417316ff9a37ad5Daniel Dunbar def __new__(cls, name, isFailure): 128253cc047de523fd2097e4d85417316ff9a37ad5Daniel Dunbar res = cls._instances.get(name) 138253cc047de523fd2097e4d85417316ff9a37ad5Daniel Dunbar if res is None: 148253cc047de523fd2097e4d85417316ff9a37ad5Daniel Dunbar cls._instances[name] = res = super(ResultCode, cls).__new__(cls) 158253cc047de523fd2097e4d85417316ff9a37ad5Daniel Dunbar return res 168253cc047de523fd2097e4d85417316ff9a37ad5Daniel Dunbar def __getnewargs__(self): 178253cc047de523fd2097e4d85417316ff9a37ad5Daniel Dunbar return (self.name, self.isFailure) 188253cc047de523fd2097e4d85417316ff9a37ad5Daniel Dunbar 19be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar def __init__(self, name, isFailure): 20be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar self.name = name 21be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar self.isFailure = isFailure 22be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar 2383a4cb51ce6803782ccc9c6ecd86b63971b5c00bDaniel Dunbar def __repr__(self): 2483a4cb51ce6803782ccc9c6ecd86b63971b5c00bDaniel Dunbar return '%s%r' % (self.__class__.__name__, 2583a4cb51ce6803782ccc9c6ecd86b63971b5c00bDaniel Dunbar (self.name, self.isFailure)) 2683a4cb51ce6803782ccc9c6ecd86b63971b5c00bDaniel Dunbar 27ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel DunbarPASS = ResultCode('PASS', False) 28ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel DunbarXFAIL = ResultCode('XFAIL', False) 29ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel DunbarFAIL = ResultCode('FAIL', True) 30ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel DunbarXPASS = ResultCode('XPASS', True) 31ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel DunbarUNRESOLVED = ResultCode('UNRESOLVED', True) 32ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel DunbarUNSUPPORTED = ResultCode('UNSUPPORTED', False) 33ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbar 34ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar# Test metric values. 35ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar 36ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbarclass MetricValue(object): 37ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar def format(self): 382849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar """ 392849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar format() -> str 402849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar 412849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar Convert this metric to a string suitable for displaying as part of the 422849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar console output. 432849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar """ 442849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar raise RuntimeError("abstract method") 452849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar 462849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar def todata(self): 472849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar """ 482849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar todata() -> json-serializable data 492849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar 502849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar Convert this metric to content suitable for serializing in the JSON test 512849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar output. 522849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar """ 53ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar raise RuntimeError("abstract method") 54ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar 55ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbarclass IntMetricValue(MetricValue): 56ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar def __init__(self, value): 57ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar self.value = value 58ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar 59ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar def format(self): 60ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar return str(self.value) 61ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar 622849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar def todata(self): 632849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar return self.value 642849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar 65ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbarclass RealMetricValue(MetricValue): 66ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar def __init__(self, value): 67ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar self.value = value 68ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar 69ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar def format(self): 70ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar return '%.4f' % self.value 71ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar 722849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar def todata(self): 732849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar return self.value 742849503ab240a2dab6f2e3c5a029e3416165554fDaniel Dunbar 75ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar# Test results. 76ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar 77ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbarclass Result(object): 78ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbar """Wrapper for the results of executing an individual test.""" 79ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbar 80d3bf8a2c0aefe36cb4bfc6f41983aa09caae2acbDaniel Dunbar def __init__(self, code, output='', elapsed=None): 81ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbar # The result code. 82ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbar self.code = code 83ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbar # The test output. 84ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbar self.output = output 85ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbar # The wall timing to execute the test, if timing. 86ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbar self.elapsed = elapsed 87ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar # The metrics reported by this test. 88ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar self.metrics = {} 89ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar 90ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar def addMetric(self, name, value): 91ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar """ 92ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar addMetric(name, value) 93ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar 94ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar Attach a test metric to the test result, with the given name and list of 95ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar values. It is an error to attempt to attach the metrics with the same 96ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar name multiple times. 97ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar 98ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar Each value must be an instance of a MetricValue subclass. 99ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar """ 100ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar if name in self.metrics: 101ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar raise ValueError("result already includes metrics for %r" % ( 102ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar name,)) 103ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar if not isinstance(value, MetricValue): 104ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar raise TypeError("unexpected metric value: %r" % (value,)) 105ff058f0a701b601f1593f2a9c8030acb652fdba6Daniel Dunbar self.metrics[name] = value 106be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar 107be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar# Test classes. 108be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar 109be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbarclass TestSuite: 110be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar """TestSuite - Information on a group of tests. 111be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar 112be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar A test suite groups together a set of logically related tests. 113be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar """ 114be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar 115be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar def __init__(self, name, source_root, exec_root, config): 116be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar self.name = name 117be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar self.source_root = source_root 118be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar self.exec_root = exec_root 119be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar # The test suite configuration. 120be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar self.config = config 121be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar 122be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar def getSourcePath(self, components): 123be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar return os.path.join(self.source_root, *components) 124be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar 125be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar def getExecPath(self, components): 126be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar return os.path.join(self.exec_root, *components) 127be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar 128be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbarclass Test: 129be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar """Test - Information on a single test instance.""" 130be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar 13136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines def __init__(self, suite, path_in_suite, config, file_path = None): 132be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar self.suite = suite 133be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar self.path_in_suite = path_in_suite 134be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar self.config = config 13536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines self.file_path = file_path 136f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar # A list of conditions under which this test is expected to fail. These 137f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar # can optionally be provided by test format handlers, and will be 138f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar # honored when the test result is supplied. 139f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar self.xfails = [] 140ccd21b26dd16c6dff207b3ded3df2bb90f1b9e6eDaniel Dunbar # The test result, once complete. 141be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar self.result = None 142be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar 143d3bf8a2c0aefe36cb4bfc6f41983aa09caae2acbDaniel Dunbar def setResult(self, result): 144d3bf8a2c0aefe36cb4bfc6f41983aa09caae2acbDaniel Dunbar if self.result is not None: 145d3bf8a2c0aefe36cb4bfc6f41983aa09caae2acbDaniel Dunbar raise ArgumentError("test result already set") 146d3bf8a2c0aefe36cb4bfc6f41983aa09caae2acbDaniel Dunbar if not isinstance(result, Result): 147d3bf8a2c0aefe36cb4bfc6f41983aa09caae2acbDaniel Dunbar raise ArgumentError("unexpected result type") 148d3bf8a2c0aefe36cb4bfc6f41983aa09caae2acbDaniel Dunbar 149d3bf8a2c0aefe36cb4bfc6f41983aa09caae2acbDaniel Dunbar self.result = result 150be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar 151f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar # Apply the XFAIL handling to resolve the result exit code. 152f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar if self.isExpectedToFail(): 153f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar if self.result.code == PASS: 154f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar self.result.code = XPASS 155f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar elif self.result.code == FAIL: 156f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar self.result.code = XFAIL 157f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar 158be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar def getFullName(self): 159eab04fc0be276db84e322dd07d371fae0161f241Daniel Dunbar return self.suite.config.name + ' :: ' + '/'.join(self.path_in_suite) 160be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar 16136b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines def getFilePath(self): 16236b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines if self.file_path: 16336b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return self.file_path 16436b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines return self.getSourcePath() 16536b56886974eae4f9c5ebc96befd3e7bfe5de338Stephen Hines 166be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar def getSourcePath(self): 167be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar return self.suite.getSourcePath(self.path_in_suite) 168be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar 169be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar def getExecPath(self): 170be7ada718139b8c840a38ba34c4af492b6a05f9fDaniel Dunbar return self.suite.getExecPath(self.path_in_suite) 171f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar 172f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar def isExpectedToFail(self): 173f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar """ 174f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar isExpectedToFail() -> bool 175f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar 176f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar Check whether this test is expected to fail in the current 177f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar configuration. This check relies on the test xfails property which by 178f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar some test formats may not be computed until the test has first been 179f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar executed. 180f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar """ 181f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar 182f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar # Check if any of the xfails match an available feature or the target. 183f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar for item in self.xfails: 184f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar # If this is the wildcard, it always fails. 185f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar if item == '*': 186f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar return True 187f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar 188f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar # If this is an exact match for one of the features, it fails. 189f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar if item in self.config.available_features: 190f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar return True 191f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar 192f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar # If this is a part of the target triple, it fails. 193f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar if item in self.suite.config.target_triple: 194f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar return True 195f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar 196f72bc792264941d1cdffe99775ce5e28bcdc51a4Daniel Dunbar return False 197