check-xml-test-suite.py revision 55253e21f0f41f03afb4842f560a852658077a89 
1#!/usr/bin/python
2import sys
3import os
4import string
5sys.path.append("python")
6import libxml2
7
8#
9# the testsuite description
10#
11CONF="xml-test-suite/xmlconf/xmlconf.xml"
12LOG="check-xml-test-suite.log"
13
14log = open(LOG, "w")
15
16#
17# Error and warning handlers
18#
19error_nr = 0
20error_msg = ''
21def errorHandler(ctx, str):
22    global error_nr
23    global error_msg
24
25    if string.find(str, "error:") >= 0:
26	error_nr = error_nr + 1
27    if len(error_msg) < 300:
28        if len(error_msg) == 0 or error_msg[-1] == '\n':
29	    error_msg = error_msg + "   >>" + str
30	else:
31	    error_msg = error_msg + str
32
33libxml2.registerErrorHandler(errorHandler, None)
34
35#warning_nr = 0
36#warning = ''
37#def warningHandler(ctx, str):
38#    global warning_nr
39#    global warning
40#
41#    warning_nr = warning_nr + 1
42#    warning = warning + str
43#
44#libxml2.registerWarningHandler(warningHandler, None)
45
46#
47# Used to load the XML testsuite description
48#
49def loadNoentDoc(filename):
50    ctxt = libxml2.createFileParserCtxt(filename)
51    if ctxt == None:
52        return None
53    ctxt.replaceEntities(1)
54    ctxt.parseDocument()
55    doc = ctxt.doc()
56    if ctxt.wellFormed() != 1:
57        doc.freeDoc()
58	return None
59    return doc
60
61#
62# The conformance testing routines
63#
64
65def testNotWf(filename, id):
66    global error_nr
67    global error_msg
68    global log
69
70    error_nr = 0
71    error_msg = ''
72
73    ctxt = libxml2.createFileParserCtxt(filename)
74    if ctxt == None:
75        return -1
76    ctxt.parseDocument()
77
78    doc = ctxt.doc()
79    if error_nr == 0 or ctxt.wellFormed() != 0:
80        print "%s: error: Well Formedness error not detected" % (id)
81	log.write("%s: error: Well Formedness error not detected\n" % (id))
82	doc.freeDoc()
83	return 0
84    return 1
85
86def testNotWfEnt(filename, id):
87    global error_nr
88    global error_msg
89    global log
90
91    error_nr = 0
92    error_msg = ''
93
94    ctxt = libxml2.createFileParserCtxt(filename)
95    if ctxt == None:
96        return -1
97    ctxt.replaceEntities(1)
98    ctxt.parseDocument()
99
100    doc = ctxt.doc()
101    if error_nr == 0 or ctxt.wellFormed() != 0:
102        print "%s: error: Well Formedness error not detected" % (id)
103	log.write("%s: error: Well Formedness error not detected\n" % (id))
104	doc.freeDoc()
105	return 0
106    return 1
107
108def testNotWfEntDtd(filename, id):
109    global error_nr
110    global error_msg
111    global log
112
113    error_nr = 0
114    error_msg = ''
115
116    ctxt = libxml2.createFileParserCtxt(filename)
117    if ctxt == None:
118        return -1
119    ctxt.replaceEntities(1)
120    ctxt.loadSubset(1)
121    ctxt.parseDocument()
122
123    doc = ctxt.doc()
124    if error_nr == 0 or ctxt.wellFormed() != 0:
125        print "%s: error: Well Formedness error not detected" % (id)
126	log.write("%s: error: Well Formedness error not detected\n" % (id))
127	doc.freeDoc()
128	return 0
129    return 1
130
131def testWfEntDtd(filename, id):
132    global error_nr
133    global error_msg
134    global log
135
136    error_nr = 0
137    error_msg = ''
138
139    ctxt = libxml2.createFileParserCtxt(filename)
140    if ctxt == None:
141        return -1
142    ctxt.replaceEntities(1)
143    ctxt.loadSubset(1)
144    ctxt.parseDocument()
145
146    doc = ctxt.doc()
147    if ctxt.wellFormed() == 0:
148        print "%s: error: wrongly failed to parse the document" % (id)
149	log.write("%s: error: wrongly failed to parse the document\n" % (id))
150	return 0
151    if error_nr != 0:
152        print "%s: warning: WF document generated an error msg" % (id)
153	log.write("%s: error: WF document generated an error msg\n" % (id))
154	doc.freeDoc()
155	return 2
156    doc.freeDoc()
157    return 1
158
159def testError(filename, id):
160    global error_nr
161    global error_msg
162    global log
163
164    error_nr = 0
165    error_msg = ''
166
167    ctxt = libxml2.createFileParserCtxt(filename)
168    if ctxt == None:
169        return -1
170    ctxt.replaceEntities(1)
171    ctxt.loadSubset(1)
172    ctxt.parseDocument()
173
174    doc = ctxt.doc()
175    if ctxt.wellFormed() == 0:
176        print "%s: warning: failed to parse the document but accepted" % (id)
177	log.write("%s: warning: failed to parse the document but accepte\n" % (id))
178	return 2
179    if error_nr != 0:
180        print "%s: warning: WF document generated an error msg" % (id)
181	log.write("%s: error: WF document generated an error msg\n" % (id))
182	doc.freeDoc()
183	return 2
184    doc.freeDoc()
185    return 1
186
187def testInvalid(filename, id):
188    global error_nr
189    global error_msg
190    global log
191
192    error_nr = 0
193    error_msg = ''
194
195    ctxt = libxml2.createFileParserCtxt(filename)
196    if ctxt == None:
197        return -1
198    ctxt.validate(1)
199    ctxt.parseDocument()
200
201    doc = ctxt.doc()
202    valid = ctxt.isValid()
203    if doc == None:
204        print "%s: error: wrongly failed to parse the document" % (id)
205	log.write("%s: error: wrongly failed to parse the document\n" % (id))
206	return 0
207    if valid == 1:
208        print "%s: error: Validity error not detected" % (id)
209	log.write("%s: error: Validity error not detected\n" % (id))
210	doc.freeDoc()
211	return 0
212    if error_nr == 0:
213        print "%s: warning: Validity error not reported" % (id)
214	log.write("%s: warning: Validity error not reported\n" % (id))
215	doc.freeDoc()
216	return 2
217
218    doc.freeDoc()
219    return 1
220
221def testValid(filename, id):
222    global error_nr
223    global error_msg
224
225    error_nr = 0
226    error_msg = ''
227
228    ctxt = libxml2.createFileParserCtxt(filename)
229    if ctxt == None:
230        return -1
231    ctxt.validate(1)
232    ctxt.parseDocument()
233
234    doc = ctxt.doc()
235    valid = ctxt.isValid()
236    if doc == None:
237        print "%s: error: wrongly failed to parse the document" % (id)
238	log.write("%s: error: wrongly failed to parse the document\n" % (id))
239	return 0
240    if valid != 1:
241        print "%s: error: Validity check failed" % (id)
242	log.write("%s: error: Validity check failed\n" % (id))
243	doc.freeDoc()
244	return 0
245    if error_nr != 0 or valid != 1:
246        print "%s: warning: valid document reported an error" % (id)
247	log.write("%s: warning: valid document reported an error\n" % (id))
248	doc.freeDoc()
249	return 2
250    doc.freeDoc()
251    return 1
252
253test_nr = 0
254test_succeed = 0
255test_failed = 0
256test_error = 0
257def runTest(test):
258    global test_nr
259    global test_failed
260    global test_error
261    global test_succeed
262    global error_msg
263    global log
264
265    uri = test.prop('URI')
266    id = test.prop('ID')
267    if uri == None:
268        print "Test without ID:", uri
269	return -1
270    if id == None:
271        print "Test without URI:", id
272	return -1
273    base = test.getBase(None)
274    URI = libxml2.buildURI(uri, base)
275    if os.access(URI, os.R_OK) == 0:
276        print "Test %s missing: base %s uri %s" % (URI, base, uri)
277	return -1
278    type = test.prop('TYPE')
279    if type == None:
280        print "Test %s missing TYPE" % (id)
281	return -1
282
283    extra = None
284    if type == "invalid":
285        res = testInvalid(URI, id)
286    elif type == "valid":
287        res = testValid(URI, id)
288    elif type == "not-wf":
289        extra =  test.prop('ENTITIES')
290	# print URI
291	#if extra == None:
292	#    res = testNotWfEntDtd(URI, id)
293 	#elif extra == 'none':
294	#    res = testNotWf(URI, id)
295	#elif extra == 'general':
296	#    res = testNotWfEnt(URI, id)
297	#elif extra == 'both' or extra == 'parameter':
298	res = testNotWfEntDtd(URI, id)
299	#else:
300	#    print "Unknow value %s for an ENTITIES test value" % (extra)
301	#    return -1
302    elif type == "error":
303	res = testError(URI, id)
304    else:
305        # TODO skipped for now
306	return -1
307
308    test_nr = test_nr + 1
309    if res > 0:
310	test_succeed = test_succeed + 1
311    elif res == 0:
312	test_failed = test_failed + 1
313    elif res < 0:
314	test_error = test_error + 1
315
316    # Log the ontext
317    if res != 1:
318	log.write("   File: %s\n" % (URI))
319	content = string.strip(test.content)
320	while content[-1] == '\n':
321	    content = content[0:-1]
322	if extra != None:
323	    log.write("   %s:%s:%s\n" % (type, extra, content))
324	else:
325	    log.write("   %s:%s\n\n" % (type, content))
326	if error_msg != '':
327	    log.write("   ----\n%s   ----\n" % (error_msg))
328	    error_msg = ''
329	log.write("\n")
330
331    return 0
332
333
334def runTestCases(case):
335    profile = case.prop('PROFILE')
336    if profile != None and \
337       string.find(profile, "IBM XML Conformance Test Suite - Production") < 0:
338	print "=>", profile
339    test = case.children
340    while test != None:
341        if test.name == 'TEST':
342	    runTest(test)
343	if test.name == 'TESTCASES':
344	    runTestCases(test)
345        test = test.next
346
347conf = loadNoentDoc(CONF)
348if conf == None:
349    print "Unable to load %s" % CONF
350    sys.exit(1)
351
352testsuite = conf.getRootElement()
353if testsuite.name != 'TESTSUITE':
354    print "Expecting TESTSUITE root element: aborting"
355    sys.exit(1)
356
357profile = testsuite.prop('PROFILE')
358if profile != None:
359    print profile
360
361case = testsuite.children
362while case != None:
363    global test_nr
364    global test_succeed
365    global test_failed
366    global test_error
367
368    if case.name == 'TESTCASES':
369	old_test_nr = test_nr
370	old_test_succeed = test_succeed
371	old_test_failed = test_failed
372	old_test_error = test_error
373        runTestCases(case)
374	print "   Ran %d tests: %d suceeded, %d failed and %d generated an error" % (
375	       test_nr - old_test_nr, test_succeed - old_test_succeed,
376	       test_failed - old_test_failed, test_error - old_test_error)
377    case = case.next
378
379conf.freeDoc()
380log.close()
381
382print "Ran %d tests: %d suceeded, %d failed and %d generated an error" % (
383      test_nr, test_succeed, test_failed, test_error)
384