1/*
2* Copyright 2006 Sony Computer Entertainment Inc.
3*
4* Licensed under the MIT Open Source License, for details please see license.txt or the website
5* http://www.opensource.org/licenses/mit-license.php
6*
7*/
8
9// The user can choose whether or not to include libxml support in the DOM. Supporting libxml will
10// require linking against it. By default libxml support is included.
11#if defined(DOM_INCLUDE_LIBXML)
12
13// This is a rework of the XML plugin that contains a complete interface to libxml2 "readXML"
14// This is intended to be a seperate plugin but I'm starting out by rewriting it in daeLIBXMLPlugin
15// because I'm not sure if all the plugin handling stuff has been tested.  Once I get a working
16// plugin I'll look into renaming it so the old daeLIBXMLPlugin can coexist with it.
17//
18#include <string>
19#include <sstream>
20#include <modules/daeLIBXMLPlugin.h>
21#include <dae.h>
22#include <dom.h>
23#include <dae/daeDatabase.h>
24#include <dae/daeMetaElement.h>
25#include <libxml/xmlreader.h>
26#include <libxml/xmlwriter.h>
27#include <libxml/xmlmemory.h>
28#include <dae/daeErrorHandler.h>
29#include <dae/daeMetaElementAttribute.h>
30
31using namespace std;
32
33
34// Some helper functions for working with libxml
35namespace {
36	daeInt getCurrentLineNumber(xmlTextReaderPtr reader) {
37#if LIBXML_VERSION >= 20620
38	return xmlTextReaderGetParserLineNumber(reader);
39#else
40	return -1;
41#endif
42	}
43
44	// Return value should be freed by caller with delete[]. Passed in value should not
45	// be null.
46	xmlChar* utf8ToLatin1(const xmlChar* utf8) {
47		int inLen = xmlStrlen(utf8);
48		int outLen = (inLen+1) * 2;
49		xmlChar* latin1 = new xmlChar[outLen];
50		int numBytes = UTF8Toisolat1(latin1, &outLen, utf8, &inLen);
51		if (numBytes < 0)
52			// Failed. Return an empty string instead.
53			numBytes = 0;
54
55		latin1[numBytes] = '\0';
56		return latin1;
57	}
58
59	// Return value should be freed by caller with delete[].
60	xmlChar* latin1ToUtf8(const string& latin1) {
61		int inLen = (int)latin1.length();
62		int outLen = (inLen+1) * 2;
63		xmlChar* utf8 = new xmlChar[outLen];
64		int numBytes = isolat1ToUTF8(utf8, &outLen, (xmlChar*)latin1.c_str(), &inLen);
65		if (numBytes < 0)
66			// Failed. Return an empty string instead.
67			numBytes = 0;
68
69		utf8[numBytes] = '\0';
70		return utf8;
71	}
72
73	typedef pair<daeString, daeString> stringPair;
74
75	// The attributes vector passed in should be empty. If 'encoding' is anything
76	// other than utf8 the caller should free the returned attribute value
77	// strings. The 'freeAttrValues' function is provided for that purpose.
78	void packageCurrentAttributes(xmlTextReaderPtr reader,
79	                              DAE::charEncoding encoding,
80	                              /* out */ vector<stringPair>& attributes) {
81		int numAttributes = xmlTextReaderAttributeCount(reader);
82		if (numAttributes == -1 || numAttributes == 0)
83			return;
84		attributes.reserve(numAttributes);
85
86		while (xmlTextReaderMoveToNextAttribute(reader) == 1) {
87			const xmlChar* xmlName = xmlTextReaderConstName(reader);
88			const xmlChar* xmlValue = xmlTextReaderConstValue(reader);
89			if (encoding == DAE::Latin1)
90				attributes.push_back(stringPair((daeString)xmlName, (daeString)utf8ToLatin1(xmlValue)));
91			else
92				attributes.push_back(stringPair((daeString)xmlName, (daeString)xmlValue));
93		}
94	}
95
96	void freeAttrValues(vector<stringPair>& pairs) {
97		for(size_t i=0, size=pairs.size(); i<size; ++i) {
98			delete[] pairs[i].second;
99			pairs[i].second = 0;
100		}
101	}
102}
103
104daeLIBXMLPlugin::daeLIBXMLPlugin(DAE& dae) : dae(dae), rawRelPath(dae)
105{
106	supportedProtocols.push_back("*");
107	xmlInitParser();
108	rawFile = NULL;
109	rawByteCount = 0;
110	saveRawFile = false;
111}
112
113daeLIBXMLPlugin::~daeLIBXMLPlugin()
114{
115	 xmlCleanupParser();
116}
117
118daeInt daeLIBXMLPlugin::setOption( daeString option, daeString value )
119{
120	if ( strcmp( option, "saveRawBinary" ) == 0 )
121	{
122		if ( strcmp( value, "true" ) == 0 || strcmp( value, "TRUE" ) == 0 )
123		{
124			saveRawFile = true;
125		}
126		else
127		{
128			saveRawFile = false;
129		}
130		return DAE_OK;
131	}
132	return DAE_ERR_INVALID_CALL;
133}
134
135daeString daeLIBXMLPlugin::getOption( daeString option )
136{
137	if ( strcmp( option, "saveRawBinary" ) == 0 )
138	{
139		if ( saveRawFile )
140		{
141			return "true";
142		}
143		return "false";
144	}
145	return NULL;
146}
147
148namespace {
149	void libxmlErrorHandler(void* arg,
150	                        const char* msg,
151	                        xmlParserSeverities severity,
152	                        xmlTextReaderLocatorPtr locator) {
153		if(severity == XML_PARSER_SEVERITY_VALIDITY_WARNING  ||
154		   severity == XML_PARSER_SEVERITY_WARNING) {
155			daeErrorHandler::get()->handleWarning(msg);
156		}
157		else
158			daeErrorHandler::get()->handleError(msg);
159	}
160}
161
162// A simple structure to help alloc/free xmlTextReader objects
163struct xmlTextReaderHelper {
164	xmlTextReaderHelper(const daeURI& uri) {
165		if((reader = xmlReaderForFile(cdom::fixUriForLibxml(uri.str()).c_str(), NULL, 0)))
166		   xmlTextReaderSetErrorHandler(reader, libxmlErrorHandler, NULL);
167	}
168
169	xmlTextReaderHelper(daeString buffer, const daeURI& baseUri) {
170		if((reader = xmlReaderForDoc((xmlChar*)buffer, cdom::fixUriForLibxml(baseUri.str()).c_str(), NULL, 0)))
171			xmlTextReaderSetErrorHandler(reader, libxmlErrorHandler, NULL);
172	};
173
174	~xmlTextReaderHelper() {
175		if (reader)
176			xmlFreeTextReader(reader);
177	}
178
179	xmlTextReaderPtr reader;
180};
181
182daeElementRef daeLIBXMLPlugin::readFromFile(const daeURI& uri) {
183	xmlTextReaderHelper readerHelper(uri);
184	if (!readerHelper.reader) {
185		daeErrorHandler::get()->handleError((string("Failed to open ") + uri.str() +
186		                                    " in daeLIBXMLPlugin::readFromFile\n").c_str());
187		return NULL;
188	}
189	return read(readerHelper.reader);
190}
191
192daeElementRef daeLIBXMLPlugin::readFromMemory(daeString buffer, const daeURI& baseUri) {
193	xmlTextReaderHelper readerHelper(buffer, baseUri);
194	if (!readerHelper.reader) {
195		daeErrorHandler::get()->handleError("Failed to open XML document from memory buffer in "
196		                                    "daeLIBXMLPlugin::readFromMemory\n");
197		return NULL;
198	}
199	return read(readerHelper.reader);
200}
201
202daeElementRef daeLIBXMLPlugin::read(_xmlTextReader* reader) {
203	// Drop everything up to the first element. In the future, we should try to store header comments somewhere.
204	while(xmlTextReaderNodeType(reader) != XML_READER_TYPE_ELEMENT)
205	{
206		if (xmlTextReaderRead(reader) != 1) {
207			daeErrorHandler::get()->handleError("Error parsing XML in daeLIBXMLPlugin::read\n");
208			return NULL;
209		}
210	}
211
212	int readRetVal = 0;
213	return readElement(reader, NULL, readRetVal);
214}
215
216daeElementRef daeLIBXMLPlugin::readElement(_xmlTextReader* reader,
217                                           daeElement* parentElement,
218                                           /* out */ int& readRetVal) {
219	assert(xmlTextReaderNodeType(reader) == XML_READER_TYPE_ELEMENT);
220	daeString elementName = (daeString)xmlTextReaderConstName(reader);
221	bool empty = xmlTextReaderIsEmptyElement(reader) != 0;
222
223	vector<attrPair> attributes;
224	packageCurrentAttributes(reader, dae.getCharEncoding(), /* out */ attributes);
225
226	daeElementRef element = beginReadElement(parentElement, elementName, attributes, getCurrentLineNumber(reader));
227	if (dae.getCharEncoding() != DAE::Utf8)
228		freeAttrValues(attributes);
229
230	if (!element) {
231		// We couldn't create the element. beginReadElement already printed an error message. Just make sure
232		// to skip ahead past the bad element.
233		xmlTextReaderNext(reader);
234		return NULL;
235	}
236
237	if ((readRetVal = xmlTextReaderRead(reader)) == -1)
238		return NULL;
239	if (empty)
240		return element;
241
242	int nodeType = xmlTextReaderNodeType(reader);
243	while (readRetVal == 1  &&  nodeType != XML_READER_TYPE_END_ELEMENT) {
244		if (nodeType == XML_READER_TYPE_ELEMENT) {
245			element->placeElement(readElement(reader, element, readRetVal));
246		}
247		else if (nodeType == XML_READER_TYPE_TEXT) {
248			const xmlChar* xmlText = xmlTextReaderConstValue(reader);
249			if (dae.getCharEncoding() == DAE::Latin1)
250				xmlText = utf8ToLatin1(xmlText);
251			readElementText(element, (daeString)xmlText, getCurrentLineNumber(reader));
252			if (dae.getCharEncoding() == DAE::Latin1)
253				delete[] xmlText;
254
255			readRetVal = xmlTextReaderRead(reader);
256		}
257		else
258			readRetVal = xmlTextReaderRead(reader);
259
260		nodeType = xmlTextReaderNodeType(reader);
261	}
262
263	if (nodeType == XML_READER_TYPE_END_ELEMENT)
264		readRetVal = xmlTextReaderRead(reader);
265
266	if (readRetVal == -1) // Something went wrong (bad xml probably)
267		return NULL;
268
269	return element;
270}
271
272daeInt daeLIBXMLPlugin::write(const daeURI& name, daeDocument *document, daeBool replace)
273{
274	// Make sure database and document are both set
275	if (!database)
276		return DAE_ERR_INVALID_CALL;
277	if(!document)
278		return DAE_ERR_COLLECTION_DOES_NOT_EXIST;
279
280	// Convert the URI to a file path, to see if we're about to overwrite a file
281	string file = cdom::uriToNativePath(name.str());
282	if (file.empty()  &&  saveRawFile)
283	{
284		daeErrorHandler::get()->handleError( "can't get path in write\n" );
285		return DAE_ERR_BACKEND_IO;
286	}
287
288	// If replace=false, don't replace existing files
289	if(!replace)
290	{
291		// Using "stat" would be better, but it's not available on all platforms
292		FILE *tempfd = fopen(file.c_str(), "r");
293		if(tempfd != NULL)
294		{
295			// File exists, return error
296			fclose(tempfd);
297			return DAE_ERR_BACKEND_FILE_EXISTS;
298		}
299		fclose(tempfd);
300	}
301	if ( saveRawFile )
302	{
303		string rawFilePath = file + ".raw";
304		if ( !replace )
305		{
306			rawFile = fopen(rawFilePath.c_str(), "rb" );
307			if ( rawFile != NULL )
308			{
309				fclose(rawFile);
310				return DAE_ERR_BACKEND_FILE_EXISTS;
311			}
312			fclose(rawFile);
313		}
314		rawFile = fopen(rawFilePath.c_str(), "wb");
315		if ( rawFile == NULL )
316		{
317			return DAE_ERR_BACKEND_IO;
318		}
319		rawRelPath.set(cdom::nativePathToUri(rawFilePath));
320		rawRelPath.makeRelativeTo( &name );
321	}
322
323	// Open the file we will write to
324	writer = xmlNewTextWriterFilename(cdom::fixUriForLibxml(name.str()).c_str(), 0);
325	if ( !writer ) {
326		ostringstream msg;
327		msg << "daeLIBXMLPlugin::write(" << name.str() << ") failed\n";
328		daeErrorHandler::get()->handleError(msg.str().c_str());
329		return DAE_ERR_BACKEND_IO;
330	}
331	xmlTextWriterSetIndentString( writer, (const xmlChar*)"\t" ); // Don't change this to spaces
332	xmlTextWriterSetIndent( writer, 1 ); // Turns indentation on
333    xmlTextWriterStartDocument( writer, "1.0", "UTF-8", NULL );
334
335	writeElement( document->getDomRoot() );
336
337	xmlTextWriterEndDocument( writer );
338	xmlTextWriterFlush( writer );
339	xmlFreeTextWriter( writer );
340
341	if ( saveRawFile && rawFile != NULL )
342	{
343		fclose( rawFile );
344	}
345
346	return DAE_OK;
347}
348
349void daeLIBXMLPlugin::writeElement( daeElement* element )
350{
351	daeMetaElement* _meta = element->getMeta();
352
353	//intercept <source> elements for special handling
354	if ( saveRawFile )
355	{
356		if ( strcmp( element->getTypeName(), "source" ) == 0 )
357		{
358			daeElementRefArray children;
359			element->getChildren( children );
360			bool validArray = false, teqCommon = false;
361			for ( unsigned int i = 0; i < children.getCount(); i++ )
362			{
363				if ( strcmp( children[i]->getTypeName(), "float_array" ) == 0 ||
364					 strcmp( children[i]->getTypeName(), "int_array" ) == 0 )
365				{
366					validArray = true;
367				}
368				else if ( strcmp( children[i]->getTypeName(), "technique_common" ) == 0 )
369				{
370					teqCommon = true;
371				}
372			}
373			if ( validArray && teqCommon )
374			{
375				writeRawSource( element );
376				return;
377			}
378		}
379	}
380
381	if (!_meta->getIsTransparent() ) {
382		xmlTextWriterStartElement(writer, (xmlChar*)element->getElementName());
383		daeMetaAttributeRefArray& attrs = _meta->getMetaAttributes();
384
385		int acnt = (int)attrs.getCount();
386
387		for(int i=0;i<acnt;i++) {
388			writeAttribute(attrs[i], element);
389		}
390	}
391	writeValue(element);
392
393	daeElementRefArray children;
394	element->getChildren( children );
395	for ( size_t x = 0; x < children.getCount(); x++ ) {
396		writeElement( children.get(x) );
397	}
398
399	/*if (_meta->getContents() != NULL) {
400		daeElementRefArray* era = (daeElementRefArray*)_meta->getContents()->getWritableMemory(element);
401		int elemCnt = (int)era->getCount();
402		for(int i = 0; i < elemCnt; i++) {
403			daeElementRef elem = (daeElementRef)era->get(i);
404			if (elem != NULL) {
405				writeElement( elem );
406			}
407		}
408	}
409	else
410	{
411		daeMetaElementAttributeArray& children = _meta->getMetaElements();
412		int cnt = (int)children.getCount();
413		for(int i=0;i<cnt;i++) {
414			daeMetaElement *type = children[i]->getElementType();
415			if ( !type->getIsAbstract() ) {
416				for (int c = 0; c < children[i]->getCount(element); c++ ) {
417					writeElement( *(daeElementRef*)children[i]->get(element,c) );
418				}
419			}
420		}
421	}*/
422	if (!_meta->getIsTransparent() ) {
423		xmlTextWriterEndElement(writer);
424	}
425}
426
427void daeLIBXMLPlugin::writeAttribute( daeMetaAttribute* attr, daeElement* element)
428{
429	ostringstream buffer;
430	attr->memoryToString(element, buffer);
431	string str = buffer.str();
432
433	// Don't write the attribute if
434	//  - The attribute isn't required AND
435	//     - The attribute has no default value and the current value is ""
436	//     - The attribute has a default value and the current value matches the default
437	if (!attr->getIsRequired()) {
438		if(!attr->getDefaultValue()  &&  str.empty())
439			return;
440		if(attr->getDefaultValue()  &&  attr->compareToDefault(element) == 0)
441			return;
442	}
443
444	xmlTextWriterStartAttribute(writer, (xmlChar*)(daeString)attr->getName());
445	xmlChar* utf8 = (xmlChar*)str.c_str();
446	if (dae.getCharEncoding() == DAE::Latin1)
447		utf8 = latin1ToUtf8(str);
448	xmlTextWriterWriteString(writer, utf8);
449	if (dae.getCharEncoding() == DAE::Latin1)
450		delete[] utf8;
451
452	xmlTextWriterEndAttribute(writer);
453}
454
455void daeLIBXMLPlugin::writeValue(daeElement* element) {
456	if (daeMetaAttribute* attr = element->getMeta()->getValueAttribute()) {
457		ostringstream buffer;
458		attr->memoryToString(element, buffer);
459		string s = buffer.str();
460		if (!s.empty()) {
461			xmlChar* str = (xmlChar*)s.c_str();
462			if (dae.getCharEncoding() == DAE::Latin1)
463				str = latin1ToUtf8(s);
464			xmlTextWriterWriteString(writer, (xmlChar*)s.c_str());
465			if (dae.getCharEncoding() == DAE::Latin1)
466				delete[] str;
467		}
468	}
469}
470
471void daeLIBXMLPlugin::writeRawSource( daeElement *src )
472{
473	daeElementRef newSrc = src->clone();
474	daeElementRef array = NULL;
475	daeElement *accessor = NULL;
476	daeElementRefArray children;
477	newSrc->getChildren( children );
478	bool isInt = false;
479	for ( int i = 0; i < (int)children.getCount(); i++ )
480	{
481		if ( strcmp( children[i]->getTypeName(), "float_array" ) == 0 )
482		{
483			array = children[i];
484			newSrc->removeChildElement( array );
485		}
486		else if ( strcmp( children[i]->getTypeName(), "int_array" ) == 0 )
487		{
488			array = children[i];
489			isInt = true;
490			newSrc->removeChildElement( array );
491		}
492		else if ( strcmp( children[i]->getTypeName(), "technique_common" ) == 0 )
493		{
494			children[i]->getChildren( children );
495		}
496		else if ( strcmp( children[i]->getTypeName(), "accessor" ) == 0 )
497		{
498			accessor = children[i];
499		}
500	}
501
502	daeULong *countPtr = (daeULong*)array->getAttributeValue( "count" );
503	daeULong count = countPtr != NULL ? *countPtr : 0;
504
505	daeULong *stridePtr = (daeULong*)accessor->getAttributeValue( "stride" );
506	daeULong stride = stridePtr != NULL ? *stridePtr : 1;
507
508	children.clear();
509	accessor->getChildren( children );
510	if ( children.getCount() > stride ) {
511		*stridePtr = children.getCount();
512	}
513
514	daeFixedName newURI;
515	sprintf( newURI, "%s#%ld", rawRelPath.getOriginalURI(), rawByteCount );
516	accessor->setAttribute( "source", newURI );
517
518	daeArray *valArray = (daeArray*)array->getValuePointer();
519
520	//TODO: pay attention to precision for the array.
521	if ( isInt )
522	{
523		for( size_t i = 0; i < count; i++ )
524		{
525			daeInt tmp = (daeInt)*(daeLong*)(valArray->getRaw(i));
526			rawByteCount += (unsigned long)(fwrite( &tmp, sizeof(daeInt), 1, rawFile ) * sizeof(daeInt));
527		}
528	}
529	else
530	{
531		for( size_t i = 0; i < count; i++ )
532		{
533			daeFloat tmp = (daeFloat)*(daeDouble*)(valArray->getRaw(i));
534			rawByteCount += (unsigned long)(fwrite( &tmp, sizeof(daeFloat), 1, rawFile ) * sizeof(daeFloat));
535		}
536	}
537
538	writeElement( newSrc );
539}
540
541#endif // DOM_INCLUDE_LIBXML
542