check-xml-test-suite.py revision fa49d87f92bdd6133f95e19b02bc214ade03099e 
1#!/usr/bin/python
2import sys
3import time
4import os
5import string
6sys.path.append("python")
7import libxml2
8
9#
10# the testsuite description
11#
12CONF="xml-test-suite/xmlconf/xmlconf.xml"
13LOG="check-xml-test-suite.log"
14
15log = open(LOG, "w")
16
17#
18# Error and warning handlers
19#
20error_nr = 0
21error_msg = ''
22def errorHandler(ctx, str):
23    global error_nr
24    global error_msg
25
26    if string.find(str, "error:") >= 0:
27	error_nr = error_nr + 1
28    if len(error_msg) < 300:
29        if len(error_msg) == 0 or error_msg[-1] == '\n':
30	    error_msg = error_msg + "   >>" + str
31	else:
32	    error_msg = error_msg + str
33
34libxml2.registerErrorHandler(errorHandler, None)
35
36#warning_nr = 0
37#warning = ''
38#def warningHandler(ctx, str):
39#    global warning_nr
40#    global warning
41#
42#    warning_nr = warning_nr + 1
43#    warning = warning + str
44#
45#libxml2.registerWarningHandler(warningHandler, None)
46
47#
48# Used to load the XML testsuite description
49#
50def loadNoentDoc(filename):
51    ctxt = libxml2.createFileParserCtxt(filename)
52    if ctxt == None:
53        return None
54    ctxt.replaceEntities(1)
55    ctxt.parseDocument()
56    try:
57	doc = ctxt.doc()
58    except:
59        doc = None
60    if ctxt.wellFormed() != 1:
61        doc.freeDoc()
62	return None
63    return doc
64
65#
66# The conformance testing routines
67#
68
69def testNotWf(filename, id):
70    global error_nr
71    global error_msg
72    global log
73
74    error_nr = 0
75    error_msg = ''
76
77    ctxt = libxml2.createFileParserCtxt(filename)
78    if ctxt == None:
79        return -1
80    ctxt.parseDocument()
81
82    try:
83	doc = ctxt.doc()
84    except:
85        doc = None
86    if error_nr == 0 or ctxt.wellFormed() != 0:
87        print "%s: error: Well Formedness error not detected" % (id)
88	log.write("%s: error: Well Formedness error not detected\n" % (id))
89	doc.freeDoc()
90	return 0
91    return 1
92
93def testNotWfEnt(filename, id):
94    global error_nr
95    global error_msg
96    global log
97
98    error_nr = 0
99    error_msg = ''
100
101    ctxt = libxml2.createFileParserCtxt(filename)
102    if ctxt == None:
103        return -1
104    ctxt.replaceEntities(1)
105    ctxt.parseDocument()
106
107    try:
108	doc = ctxt.doc()
109    except:
110        doc = None
111    if error_nr == 0 or ctxt.wellFormed() != 0:
112        print "%s: error: Well Formedness error not detected" % (id)
113	log.write("%s: error: Well Formedness error not detected\n" % (id))
114	doc.freeDoc()
115	return 0
116    return 1
117
118def testNotWfEntDtd(filename, id):
119    global error_nr
120    global error_msg
121    global log
122
123    error_nr = 0
124    error_msg = ''
125
126    ctxt = libxml2.createFileParserCtxt(filename)
127    if ctxt == None:
128        return -1
129    ctxt.replaceEntities(1)
130    ctxt.loadSubset(1)
131    ctxt.parseDocument()
132
133    try:
134	doc = ctxt.doc()
135    except:
136        doc = None
137    if error_nr == 0 or ctxt.wellFormed() != 0:
138        print "%s: error: Well Formedness error not detected" % (id)
139	log.write("%s: error: Well Formedness error not detected\n" % (id))
140	doc.freeDoc()
141	return 0
142    return 1
143
144def testWfEntDtd(filename, id):
145    global error_nr
146    global error_msg
147    global log
148
149    error_nr = 0
150    error_msg = ''
151
152    ctxt = libxml2.createFileParserCtxt(filename)
153    if ctxt == None:
154        return -1
155    ctxt.replaceEntities(1)
156    ctxt.loadSubset(1)
157    ctxt.parseDocument()
158
159    try:
160	doc = ctxt.doc()
161    except:
162        doc = None
163    if ctxt.wellFormed() == 0:
164        print "%s: error: wrongly failed to parse the document" % (id)
165	log.write("%s: error: wrongly failed to parse the document\n" % (id))
166	return 0
167    if error_nr != 0:
168        print "%s: warning: WF document generated an error msg" % (id)
169	log.write("%s: error: WF document generated an error msg\n" % (id))
170	doc.freeDoc()
171	return 2
172    doc.freeDoc()
173    return 1
174
175def testError(filename, id):
176    global error_nr
177    global error_msg
178    global log
179
180    error_nr = 0
181    error_msg = ''
182
183    ctxt = libxml2.createFileParserCtxt(filename)
184    if ctxt == None:
185        return -1
186    ctxt.replaceEntities(1)
187    ctxt.loadSubset(1)
188    ctxt.parseDocument()
189
190    try:
191	doc = ctxt.doc()
192    except:
193        doc = None
194    if ctxt.wellFormed() == 0:
195        print "%s: warning: failed to parse the document but accepted" % (id)
196	log.write("%s: warning: failed to parse the document but accepte\n" % (id))
197	return 2
198    if error_nr != 0:
199        print "%s: warning: WF document generated an error msg" % (id)
200	log.write("%s: error: WF document generated an error msg\n" % (id))
201	doc.freeDoc()
202	return 2
203    doc.freeDoc()
204    return 1
205
206def testInvalid(filename, id):
207    global error_nr
208    global error_msg
209    global log
210
211    error_nr = 0
212    error_msg = ''
213
214    ctxt = libxml2.createFileParserCtxt(filename)
215    if ctxt == None:
216        return -1
217    ctxt.validate(1)
218    ctxt.parseDocument()
219
220    try:
221	doc = ctxt.doc()
222    except:
223        doc = None
224    valid = ctxt.isValid()
225    if doc == None:
226        print "%s: error: wrongly failed to parse the document" % (id)
227	log.write("%s: error: wrongly failed to parse the document\n" % (id))
228	return 0
229    if valid == 1:
230        print "%s: error: Validity error not detected" % (id)
231	log.write("%s: error: Validity error not detected\n" % (id))
232	doc.freeDoc()
233	return 0
234    if error_nr == 0:
235        print "%s: warning: Validity error not reported" % (id)
236	log.write("%s: warning: Validity error not reported\n" % (id))
237	doc.freeDoc()
238	return 2
239
240    doc.freeDoc()
241    return 1
242
243def testValid(filename, id):
244    global error_nr
245    global error_msg
246
247    error_nr = 0
248    error_msg = ''
249
250    ctxt = libxml2.createFileParserCtxt(filename)
251    if ctxt == None:
252        return -1
253    ctxt.validate(1)
254    ctxt.parseDocument()
255
256    try:
257	doc = ctxt.doc()
258    except:
259        doc = None
260    valid = ctxt.isValid()
261    if doc == None:
262        print "%s: error: wrongly failed to parse the document" % (id)
263	log.write("%s: error: wrongly failed to parse the document\n" % (id))
264	return 0
265    if valid != 1:
266        print "%s: error: Validity check failed" % (id)
267	log.write("%s: error: Validity check failed\n" % (id))
268	doc.freeDoc()
269	return 0
270    if error_nr != 0 or valid != 1:
271        print "%s: warning: valid document reported an error" % (id)
272	log.write("%s: warning: valid document reported an error\n" % (id))
273	doc.freeDoc()
274	return 2
275    doc.freeDoc()
276    return 1
277
278test_nr = 0
279test_succeed = 0
280test_failed = 0
281test_error = 0
282def runTest(test):
283    global test_nr
284    global test_failed
285    global test_error
286    global test_succeed
287    global error_msg
288    global log
289
290    uri = test.prop('URI')
291    id = test.prop('ID')
292    if uri == None:
293        print "Test without ID:", uri
294	return -1
295    if id == None:
296        print "Test without URI:", id
297	return -1
298    base = test.getBase(None)
299    URI = libxml2.buildURI(uri, base)
300    if os.access(URI, os.R_OK) == 0:
301        print "Test %s missing: base %s uri %s" % (URI, base, uri)
302	return -1
303    type = test.prop('TYPE')
304    if type == None:
305        print "Test %s missing TYPE" % (id)
306	return -1
307
308    extra = None
309    if type == "invalid":
310        res = testInvalid(URI, id)
311    elif type == "valid":
312        res = testValid(URI, id)
313    elif type == "not-wf":
314        extra =  test.prop('ENTITIES')
315	# print URI
316	#if extra == None:
317	#    res = testNotWfEntDtd(URI, id)
318 	#elif extra == 'none':
319	#    res = testNotWf(URI, id)
320	#elif extra == 'general':
321	#    res = testNotWfEnt(URI, id)
322	#elif extra == 'both' or extra == 'parameter':
323	res = testNotWfEntDtd(URI, id)
324	#else:
325	#    print "Unknow value %s for an ENTITIES test value" % (extra)
326	#    return -1
327    elif type == "error":
328	res = testError(URI, id)
329    else:
330        # TODO skipped for now
331	return -1
332
333    test_nr = test_nr + 1
334    if res > 0:
335	test_succeed = test_succeed + 1
336    elif res == 0:
337	test_failed = test_failed + 1
338    elif res < 0:
339	test_error = test_error + 1
340
341    # Log the ontext
342    if res != 1:
343	log.write("   File: %s\n" % (URI))
344	content = string.strip(test.content)
345	while content[-1] == '\n':
346	    content = content[0:-1]
347	if extra != None:
348	    log.write("   %s:%s:%s\n" % (type, extra, content))
349	else:
350	    log.write("   %s:%s\n\n" % (type, content))
351	if error_msg != '':
352	    log.write("   ----\n%s   ----\n" % (error_msg))
353	    error_msg = ''
354	log.write("\n")
355
356    return 0
357
358
359def runTestCases(case):
360    profile = case.prop('PROFILE')
361    if profile != None and \
362       string.find(profile, "IBM XML Conformance Test Suite - Production") < 0:
363	print "=>", profile
364    test = case.children
365    while test != None:
366        if test.name == 'TEST':
367	    runTest(test)
368	if test.name == 'TESTCASES':
369	    runTestCases(test)
370        test = test.next
371
372conf = loadNoentDoc(CONF)
373if conf == None:
374    print "Unable to load %s" % CONF
375    sys.exit(1)
376
377testsuite = conf.getRootElement()
378if testsuite.name != 'TESTSUITE':
379    print "Expecting TESTSUITE root element: aborting"
380    sys.exit(1)
381
382profile = testsuite.prop('PROFILE')
383if profile != None:
384    print profile
385
386start = time.time()
387
388case = testsuite.children
389while case != None:
390    global test_nr
391    global test_succeed
392    global test_failed
393    global test_error
394
395    if case.name == 'TESTCASES':
396	old_test_nr = test_nr
397	old_test_succeed = test_succeed
398	old_test_failed = test_failed
399	old_test_error = test_error
400        runTestCases(case)
401	print "   Ran %d tests: %d suceeded, %d failed and %d generated an error" % (
402	       test_nr - old_test_nr, test_succeed - old_test_succeed,
403	       test_failed - old_test_failed, test_error - old_test_error)
404    case = case.next
405
406conf.freeDoc()
407log.close()
408
409print "Ran %d tests: %d suceeded, %d failed and %d generated an error in %.2f s." % (
410      test_nr, test_succeed, test_failed, test_error, time.time() - start)
411