check-xml-test-suite.py revision 5821806d5e7f356e8fa4b058a389a808ea183019
1#!/usr/bin/python
2import sys
3import time
4import os
5import string
6sys.path.insert(0, "python")
7import libxml2
8
9test_nr = 0
10test_succeed = 0
11test_failed = 0
12test_error = 0
13
14#
15# the testsuite description
16#
17CONF="xml-test-suite/xmlconf/xmlconf.xml"
18LOG="check-xml-test-suite.log"
19
20log = open(LOG, "w")
21
22#
23# Error and warning handlers
24#
25error_nr = 0
26error_msg = ''
27def errorHandler(ctx, str):
28    global error_nr
29    global error_msg
30
31    error_nr = error_nr + 1
32    if len(error_msg) < 300:
33        if len(error_msg) == 0 or error_msg[-1] == '\n':
34	    error_msg = error_msg + "   >>" + str
35	else:
36	    error_msg = error_msg + str
37
38libxml2.registerErrorHandler(errorHandler, None)
39
40#warning_nr = 0
41#warning = ''
42#def warningHandler(ctx, str):
43#    global warning_nr
44#    global warning
45#
46#    warning_nr = warning_nr + 1
47#    warning = warning + str
48#
49#libxml2.registerWarningHandler(warningHandler, None)
50
51#
52# Used to load the XML testsuite description
53#
54def loadNoentDoc(filename):
55    ctxt = libxml2.createFileParserCtxt(filename)
56    if ctxt == None:
57        return None
58    ctxt.replaceEntities(1)
59    ctxt.parseDocument()
60    try:
61	doc = ctxt.doc()
62    except:
63        doc = None
64    if ctxt.wellFormed() != 1:
65        doc.freeDoc()
66	return None
67    return doc
68
69#
70# The conformance testing routines
71#
72
73def testNotWf(filename, id):
74    global error_nr
75    global error_msg
76    global log
77
78    error_nr = 0
79    error_msg = ''
80
81    ctxt = libxml2.createFileParserCtxt(filename)
82    if ctxt == None:
83        return -1
84    ret = ctxt.parseDocument()
85
86    try:
87	doc = ctxt.doc()
88    except:
89        doc = None
90    if doc != None:
91	doc.freeDoc()
92    if ret == 0 or ctxt.wellFormed() != 0:
93        print "%s: error: Well Formedness error not detected" % (id)
94	log.write("%s: error: Well Formedness error not detected\n" % (id))
95	return 0
96    return 1
97
98def testNotWfEnt(filename, id):
99    global error_nr
100    global error_msg
101    global log
102
103    error_nr = 0
104    error_msg = ''
105
106    ctxt = libxml2.createFileParserCtxt(filename)
107    if ctxt == None:
108        return -1
109    ctxt.replaceEntities(1)
110    ret = ctxt.parseDocument()
111
112    try:
113	doc = ctxt.doc()
114    except:
115        doc = None
116    if doc != None:
117	doc.freeDoc()
118    if ret == 0 or ctxt.wellFormed() != 0:
119        print "%s: error: Well Formedness error not detected" % (id)
120	log.write("%s: error: Well Formedness error not detected\n" % (id))
121	return 0
122    return 1
123
124def testNotWfEntDtd(filename, id):
125    global error_nr
126    global error_msg
127    global log
128
129    error_nr = 0
130    error_msg = ''
131
132    ctxt = libxml2.createFileParserCtxt(filename)
133    if ctxt == None:
134        return -1
135    ctxt.replaceEntities(1)
136    ctxt.loadSubset(1)
137    ret = ctxt.parseDocument()
138
139    try:
140	doc = ctxt.doc()
141    except:
142        doc = None
143    if doc != None:
144	doc.freeDoc()
145    if ret == 0 or ctxt.wellFormed() != 0:
146        print "%s: error: Well Formedness error not detected" % (id)
147	log.write("%s: error: Well Formedness error not detected\n" % (id))
148	return 0
149    return 1
150
151def testWfEntDtd(filename, id):
152    global error_nr
153    global error_msg
154    global log
155
156    error_nr = 0
157    error_msg = ''
158
159    ctxt = libxml2.createFileParserCtxt(filename)
160    if ctxt == None:
161        return -1
162    ctxt.replaceEntities(1)
163    ctxt.loadSubset(1)
164    ret = ctxt.parseDocument()
165
166    try:
167	doc = ctxt.doc()
168    except:
169        doc = None
170    if doc == None or ret != 0 or ctxt.wellFormed() == 0:
171        print "%s: error: wrongly failed to parse the document" % (id)
172	log.write("%s: error: wrongly failed to parse the document\n" % (id))
173	if doc != None:
174	    doc.freeDoc()
175	return 0
176    if error_nr != 0:
177        print "%s: warning: WF document generated an error msg" % (id)
178	log.write("%s: error: WF document generated an error msg\n" % (id))
179	doc.freeDoc()
180	return 2
181    doc.freeDoc()
182    return 1
183
184def testError(filename, id):
185    global error_nr
186    global error_msg
187    global log
188
189    error_nr = 0
190    error_msg = ''
191
192    ctxt = libxml2.createFileParserCtxt(filename)
193    if ctxt == None:
194        return -1
195    ctxt.replaceEntities(1)
196    ctxt.loadSubset(1)
197    ret = ctxt.parseDocument()
198
199    try:
200	doc = ctxt.doc()
201    except:
202        doc = None
203    if doc != None:
204	doc.freeDoc()
205    if ctxt.wellFormed() == 0:
206        print "%s: warning: failed to parse the document but accepted" % (id)
207	log.write("%s: warning: failed to parse the document but accepte\n" % (id))
208	return 2
209    if error_nr != 0:
210        print "%s: warning: WF document generated an error msg" % (id)
211	log.write("%s: error: WF document generated an error msg\n" % (id))
212	return 2
213    return 1
214
215def testInvalid(filename, id):
216    global error_nr
217    global error_msg
218    global log
219
220    error_nr = 0
221    error_msg = ''
222
223    ctxt = libxml2.createFileParserCtxt(filename)
224    if ctxt == None:
225        return -1
226    ctxt.validate(1)
227    ret = ctxt.parseDocument()
228
229    try:
230	doc = ctxt.doc()
231    except:
232        doc = None
233    valid = ctxt.isValid()
234    if doc == None:
235        print "%s: error: wrongly failed to parse the document" % (id)
236	log.write("%s: error: wrongly failed to parse the document\n" % (id))
237	return 0
238    if valid == 1:
239        print "%s: error: Validity error not detected" % (id)
240	log.write("%s: error: Validity error not detected\n" % (id))
241	doc.freeDoc()
242	return 0
243    if error_nr == 0:
244        print "%s: warning: Validity error not reported" % (id)
245	log.write("%s: warning: Validity error not reported\n" % (id))
246	doc.freeDoc()
247	return 2
248
249    doc.freeDoc()
250    return 1
251
252def testValid(filename, id):
253    global error_nr
254    global error_msg
255
256    error_nr = 0
257    error_msg = ''
258
259    ctxt = libxml2.createFileParserCtxt(filename)
260    if ctxt == None:
261        return -1
262    ctxt.validate(1)
263    ctxt.parseDocument()
264
265    try:
266	doc = ctxt.doc()
267    except:
268        doc = None
269    valid = ctxt.isValid()
270    if doc == None:
271        print "%s: error: wrongly failed to parse the document" % (id)
272	log.write("%s: error: wrongly failed to parse the document\n" % (id))
273	return 0
274    if valid != 1:
275        print "%s: error: Validity check failed" % (id)
276	log.write("%s: error: Validity check failed\n" % (id))
277	doc.freeDoc()
278	return 0
279    if error_nr != 0 or valid != 1:
280        print "%s: warning: valid document reported an error" % (id)
281	log.write("%s: warning: valid document reported an error\n" % (id))
282	doc.freeDoc()
283	return 2
284    doc.freeDoc()
285    return 1
286
287def runTest(test):
288    global test_nr
289    global test_succeed
290    global test_failed
291    global error_msg
292    global log
293
294    uri = test.prop('URI')
295    id = test.prop('ID')
296    if uri == None:
297        print "Test without ID:", uri
298	return -1
299    if id == None:
300        print "Test without URI:", id
301	return -1
302    base = test.getBase(None)
303    URI = libxml2.buildURI(uri, base)
304    if os.access(URI, os.R_OK) == 0:
305        print "Test %s missing: base %s uri %s" % (URI, base, uri)
306	return -1
307    type = test.prop('TYPE')
308    if type == None:
309        print "Test %s missing TYPE" % (id)
310	return -1
311
312    extra = None
313    if type == "invalid":
314        res = testInvalid(URI, id)
315    elif type == "valid":
316        res = testValid(URI, id)
317    elif type == "not-wf":
318        extra =  test.prop('ENTITIES')
319	# print URI
320	#if extra == None:
321	#    res = testNotWfEntDtd(URI, id)
322 	#elif extra == 'none':
323	#    res = testNotWf(URI, id)
324	#elif extra == 'general':
325	#    res = testNotWfEnt(URI, id)
326	#elif extra == 'both' or extra == 'parameter':
327	res = testNotWfEntDtd(URI, id)
328	#else:
329	#    print "Unknow value %s for an ENTITIES test value" % (extra)
330	#    return -1
331    elif type == "error":
332	res = testError(URI, id)
333    else:
334        # TODO skipped for now
335	return -1
336
337    test_nr = test_nr + 1
338    if res > 0:
339	test_succeed = test_succeed + 1
340    elif res == 0:
341	test_failed = test_failed + 1
342    elif res < 0:
343	test_error = test_error + 1
344
345    # Log the ontext
346    if res != 1:
347	log.write("   File: %s\n" % (URI))
348	content = string.strip(test.content)
349	while content[-1] == '\n':
350	    content = content[0:-1]
351	if extra != None:
352	    log.write("   %s:%s:%s\n" % (type, extra, content))
353	else:
354	    log.write("   %s:%s\n\n" % (type, content))
355	if error_msg != '':
356	    log.write("   ----\n%s   ----\n" % (error_msg))
357	    error_msg = ''
358	log.write("\n")
359
360    return 0
361
362
363def runTestCases(case):
364    profile = case.prop('PROFILE')
365    if profile != None and \
366       string.find(profile, "IBM XML Conformance Test Suite - Production") < 0:
367	print "=>", profile
368    test = case.children
369    while test != None:
370        if test.name == 'TEST':
371	    runTest(test)
372	if test.name == 'TESTCASES':
373	    runTestCases(test)
374        test = test.next
375
376conf = loadNoentDoc(CONF)
377if conf == None:
378    print "Unable to load %s" % CONF
379    sys.exit(1)
380
381testsuite = conf.getRootElement()
382if testsuite.name != 'TESTSUITE':
383    print "Expecting TESTSUITE root element: aborting"
384    sys.exit(1)
385
386profile = testsuite.prop('PROFILE')
387if profile != None:
388    print profile
389
390start = time.time()
391
392case = testsuite.children
393while case != None:
394    if case.name == 'TESTCASES':
395	old_test_nr = test_nr
396	old_test_succeed = test_succeed
397	old_test_failed = test_failed
398	old_test_error = test_error
399        runTestCases(case)
400	print "   Ran %d tests: %d suceeded, %d failed and %d generated an error" % (
401	       test_nr - old_test_nr, test_succeed - old_test_succeed,
402	       test_failed - old_test_failed, test_error - old_test_error)
403    case = case.next
404
405conf.freeDoc()
406log.close()
407
408print "Ran %d tests: %d suceeded, %d failed and %d generated an error in %.2f s." % (
409      test_nr, test_succeed, test_failed, test_error, time.time() - start)
410