1f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins IIIimport sys 2f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins IIIimport os 3f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins IIIimport os.path 4f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins IIIfrom glob import glob 5f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins IIIimport optparse 6f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III 7f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins IIIVALGRIND_CMD = 'valgrind --tool=memcheck --leak-check=yes --undef-value-errors=yes ' 8f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III 9f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins IIIdef compareOutputs( expected, actual, message ): 10f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III expected = expected.strip().replace('\r','').split('\n') 11f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III actual = actual.strip().replace('\r','').split('\n') 12f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III diff_line = 0 13f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III max_line_to_compare = min( len(expected), len(actual) ) 14f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III for index in xrange(0,max_line_to_compare): 15f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III if expected[index].strip() != actual[index].strip(): 16f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III diff_line = index + 1 17f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III break 18f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III if diff_line == 0 and len(expected) != len(actual): 19f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III diff_line = max_line_to_compare+1 20f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III if diff_line == 0: 21f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III return None 22f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III def safeGetLine( lines, index ): 23f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III index += -1 24f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III if index >= len(lines): 25f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III return '' 26f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III return lines[index].strip() 27f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III return """ Difference in %s at line %d: 28f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III Expected: '%s' 29f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III Actual: '%s' 30f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III""" % (message, diff_line, 31f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III safeGetLine(expected,diff_line), 32f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III safeGetLine(actual,diff_line) ) 33f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III 34f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins IIIdef safeReadFile( path ): 35f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III try: 36f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III return file( path, 'rt' ).read() 37f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III except IOError, e: 38f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III return '<File "%s" is missing: %s>' % (path,e) 39f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III 40f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins IIIdef runAllTests( jsontest_executable_path, input_dir = None, 41f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III use_valgrind=False, with_json_checker=False ): 42f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III if not input_dir: 43f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III input_dir = os.path.join( os.getcwd(), 'data' ) 44f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III tests = glob( os.path.join( input_dir, '*.json' ) ) 45f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III if with_json_checker: 46f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III test_jsonchecker = glob( os.path.join( input_dir, '../jsonchecker', '*.json' ) ) 47f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III else: 48f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III test_jsonchecker = [] 49f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III failed_tests = [] 50f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III valgrind_path = use_valgrind and VALGRIND_CMD or '' 51f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III for input_path in tests + test_jsonchecker: 52f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III expect_failure = os.path.basename( input_path ).startswith( 'fail' ) 53f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III is_json_checker_test = (input_path in test_jsonchecker) or expect_failure 54f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III print 'TESTING:', input_path, 55f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III options = is_json_checker_test and '--json-checker' or '' 56f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III pipe = os.popen( "%s%s %s %s" % ( 57f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III valgrind_path, jsontest_executable_path, options, 58f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III input_path) ) 59f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III process_output = pipe.read() 60f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III status = pipe.close() 61f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III if is_json_checker_test: 62f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III if expect_failure: 63f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III if status is None: 64f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III print 'FAILED' 65f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III failed_tests.append( (input_path, 'Parsing should have failed:\n%s' % 66f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III safeReadFile(input_path)) ) 67f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III else: 68f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III print 'OK' 69f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III else: 70f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III if status is not None: 71f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III print 'FAILED' 72f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III failed_tests.append( (input_path, 'Parsing failed:\n' + process_output) ) 73f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III else: 74f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III print 'OK' 75f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III else: 76f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III base_path = os.path.splitext(input_path)[0] 77f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III actual_output = safeReadFile( base_path + '.actual' ) 78f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III actual_rewrite_output = safeReadFile( base_path + '.actual-rewrite' ) 79f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III file(base_path + '.process-output','wt').write( process_output ) 80f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III if status: 81f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III print 'parsing failed' 82f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III failed_tests.append( (input_path, 'Parsing failed:\n' + process_output) ) 83f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III else: 84f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III expected_output_path = os.path.splitext(input_path)[0] + '.expected' 85f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III expected_output = file( expected_output_path, 'rt' ).read() 86f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III detail = ( compareOutputs( expected_output, actual_output, 'input' ) 87f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III or compareOutputs( expected_output, actual_rewrite_output, 'rewrite' ) ) 88f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III if detail: 89f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III print 'FAILED' 90f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III failed_tests.append( (input_path, detail) ) 91f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III else: 92f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III print 'OK' 93f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III 94f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III if failed_tests: 95f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III print 96f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III print 'Failure details:' 97f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III for failed_test in failed_tests: 98f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III print '* Test', failed_test[0] 99f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III print failed_test[1] 100f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III print 101f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III print 'Test results: %d passed, %d failed.' % (len(tests)-len(failed_tests), 102f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III len(failed_tests) ) 103f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III return 1 104f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III else: 105f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III print 'All %d tests passed.' % len(tests) 106f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III return 0 107f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III 108f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins IIIdef main(): 109f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III from optparse import OptionParser 110f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III parser = OptionParser( usage="%prog [options] <path to jsontestrunner.exe> [test case directory]" ) 111f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III parser.add_option("--valgrind", 112f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III action="store_true", dest="valgrind", default=False, 113f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III help="run all the tests using valgrind to detect memory leaks") 114f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III parser.add_option("-c", "--with-json-checker", 115f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III action="store_true", dest="with_json_checker", default=False, 116f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III help="run all the tests from the official JSONChecker test suite of json.org") 117f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III parser.enable_interspersed_args() 118f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III options, args = parser.parse_args() 119f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III 120f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III if len(args) < 1 or len(args) > 2: 121f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III parser.error( 'Must provides at least path to jsontestrunner executable.' ) 122f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III sys.exit( 1 ) 123f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III 124f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III jsontest_executable_path = os.path.normpath( os.path.abspath( args[0] ) ) 125f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III if len(args) > 1: 126f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III input_path = os.path.normpath( os.path.abspath( args[1] ) ) 127f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III else: 128f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III input_path = None 129f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III status = runAllTests( jsontest_executable_path, input_path, 130f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III use_valgrind=options.valgrind, with_json_checker=options.with_json_checker ) 131f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III sys.exit( status ) 132f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III 133f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins IIIif __name__ == '__main__': 134f59fb0e83fd0a4b41700d3f5eebdc8d21b173c2eLeon Scroggins III main() 135