1/* GENERATED SOURCE. DO NOT MODIFY. */
2// © 2016 and later: Unicode, Inc. and others.
3// License & terms of use: http://www.unicode.org/copyright.html#License
4/*
5**********************************************************************
6* Copyright (c) 2002-2010, International Business Machines
7* Corporation and others.  All Rights Reserved.
8**********************************************************************
9* Author: Mark Davis
10**********************************************************************
11*/
12package android.icu.dev.test.cldr;
13
14import java.io.File;
15import java.io.IOException;
16import java.io.PrintWriter;
17import java.io.StringWriter;
18import java.text.ParseException;
19import java.util.ArrayList;
20import java.util.Collection;
21import java.util.Date;
22import java.util.HashMap;
23import java.util.Iterator;
24import java.util.List;
25import java.util.Map;
26import java.util.Set;
27import java.util.TreeMap;
28import java.util.TreeSet;
29import java.util.regex.Matcher;
30import java.util.regex.Pattern;
31
32import javax.xml.parsers.SAXParser;
33import javax.xml.parsers.SAXParserFactory;
34
35import org.junit.Ignore;
36import org.junit.Test;
37import org.xml.sax.Attributes;
38import org.xml.sax.SAXException;
39import org.xml.sax.helpers.DefaultHandler;
40
41import android.icu.dev.test.TestFmwk;
42import android.icu.text.DateFormat;
43import android.icu.text.NumberFormat;
44import android.icu.text.SimpleDateFormat;
45import android.icu.text.UTF16;
46import android.icu.text.UnicodeSet;
47import android.icu.util.Currency;
48import android.icu.util.TimeZone;
49import android.icu.util.ULocale;
50
51/**
52 * This is a test file that takes in the CLDR XML test files and test against
53 * ICU4J. This test file is used to verify that ICU4J is implemented correctly.
54 * As it stands, the test generates all the errors to the console by logging it.
55 * The logging is only possible if "-v" or verbose is set as an argument.
56 * This will allow users to know what problems occurred within CLDR and ICU.
57 * Collator was disabled in this test file and therefore will be skipped.
58 *
59 * Instructions:
60 * 1)   In order for this to work correctly, you must download the latest CLDR data
61 *      in the form of XML. You must also set the CLDR directory using:
62 *          -DCLDR_DIRECTORY=<top level of cldr>
63 * 2)   You may also consider increasing the memory using -Xmx512m.
64 * 3)   For speed purposes, you may consider creating a temporary directory for the
65 *      CLDR cache using:
66 *          -DCLDR_DTD_CACHE=<cldr cache directory>
67 * 4)   You may use other environment variables to narrow down your tests using:
68 *          -DXML_MATCH=".*"
69 *              -DXML_MATCH="de.*"  (or whatever regex you want) to just test certain locales.
70 *          -DTEST_MATCH="zone.*"   (or whatever regex you want) to just test collation, numbers, etc.
71 *          -DZONE_MATCH="(?!America/Argentina).*"
72 *              -DZONE_MATCH=".*Moscow.*" (to only test certain zones)
73
74 * @author medavis
75 * @author John Huan Vu (johnvu@us.ibm.com)
76 */
77public class TestCLDRVsICU extends TestFmwk {
78    static final boolean DEBUG = false;
79
80    // ULocale uLocale = ULocale.ENGLISH;
81    // Locale oLocale = Locale.ENGLISH; // TODO Drop once ICU4J has ULocale everywhere
82    // static PrintWriter log;
83    SAXParser SAX;
84    static Matcher LOCALE_MATCH, TEST_MATCH, ZONE_MATCH;
85    static String CLDR_DIRECTORY;
86    static {
87        System.out.println();
88        LOCALE_MATCH = getEnvironmentRegex("XML_MATCH", ".*");
89        TEST_MATCH = getEnvironmentRegex("TEST_MATCH", ".*");
90        ZONE_MATCH = getEnvironmentRegex("ZONE_MATCH", ".*");
91
92        // CLDR_DIRECTORY is where all the CLDR XML test files are located
93        // WARNING: THIS IS TEMPORARY DIRECTORY UNTIL THE FILES ARE STRAIGHTENED OUT
94        CLDR_DIRECTORY = getEnvironmentString("CLDR_DIRECTORY", "C:\\Unicode-CVS2\\cldr\\");
95        System.out.println();
96    }
97
98    private static Matcher getEnvironmentRegex(String key, String defaultValue) {
99        return Pattern.compile(getEnvironmentString(key, defaultValue)).matcher("");
100    }
101
102    private static String getEnvironmentString(String key, String defaultValue) {
103        String temp = System.getProperty(key);
104        if (temp == null)
105            temp = defaultValue;
106        else
107            System.out.print("-D" + key + "=\"" + temp + "\" ");
108        return temp;
109    }
110
111    Set allLocales = new TreeSet();
112
113    // TODO(junit): seems to be failing with missing locales - maybe rewrite as parameterized
114    @Ignore
115    @Test
116    public void TestFiles() throws SAXException, IOException {
117        // only get ICU's locales
118        Set s = new TreeSet();
119        addLocales(NumberFormat.getAvailableULocales(), s);
120        addLocales(DateFormat.getAvailableULocales(), s);
121
122        // johnvu: Collator was originally disabled
123        // addLocales(Collator.getAvailableULocales(), s);
124
125        // filter, to make tracking down bugs easier
126        for (Iterator it = s.iterator(); it.hasNext();) {
127            String locale = (String) it.next();
128            if (!LOCALE_MATCH.reset(locale).matches())
129                continue;
130            _test(locale);
131        }
132    }
133
134    public void addLocales(ULocale[] list, Collection s) {
135        for (int i = 0; i < list.length; ++i) {
136            allLocales.add(list[i].toString());
137            s.add(list[i].getLanguage());
138        }
139    }
140
141    public String getLanguage(ULocale uLocale) {
142        String result = uLocale.getLanguage();
143        String script = uLocale.getScript();
144        if (script.length() != 0)
145            result += "_" + script;
146        return result;
147    }
148
149    private void _test(String localeName) throws SAXException, IOException {
150        // uLocale = new ULocale(localeName);
151        // oLocale = uLocale.toLocale();
152
153        File f = new File(CLDR_DIRECTORY, "test/" + localeName + ".xml");
154        logln("Testing " + f.getCanonicalPath());
155        SAX.parse(f, DEFAULT_HANDLER);
156    }
157
158    private static class ToHex {
159        public String transliterate(String in) {
160            StringBuilder sb = new StringBuilder();
161            for (int i = 0; i < in.length(); ++i) {
162                char c = in.charAt(i);
163                sb.append("\\u");
164                if (c < 1000) {
165                    sb.append('0');
166                    if (c < 100) {
167                        sb.append('0');
168                        if (c < 10) {
169                            sb.append('0');
170                        }
171                    }
172                }
173                sb.append(Integer.toHexString((int) c));
174            }
175            return sb.toString();
176        }
177    }
178
179    // static Transliterator toUnicode = Transliterator.getInstance("any-hex");
180    private static final ToHex toUnicode = new ToHex();
181
182    static public String showString(String in) {
183        return "\u00AB" + in + "\u00BB (" + toUnicode.transliterate(in) + ")";
184    }
185
186    // ============ SAX Handler Infrastructure ============
187
188    abstract public class Handler {
189        Map settings = new TreeMap();
190        String name;
191        List currentLocales = new ArrayList();
192        int failures = 0;
193
194        void setName(String name) {
195            this.name = name;
196        }
197
198        void set(String attributeName, String attributeValue) {
199            // if (DEBUG) logln(attributeName + " => " + attributeValue);
200            settings.put(attributeName, attributeValue);
201        }
202
203        void checkResult(String value) {
204            if (settings.get("draft").equals("unconfirmed") || settings.get("draft").equals("provisional")) {
205                return; // skip draft
206            }
207            ULocale ul = new ULocale("xx");
208            try {
209                for (int i = 0; i < currentLocales.size(); ++i) {
210                    ul = (ULocale) currentLocales.get(i);
211                    // loglnSAX("  Checking " + ul + "(" + ul.getDisplayName(ULocale.ENGLISH) + ")" + " for " + name);
212                    handleResult(ul, value);
213                    if (failures != 0) {
214                        errln("\tTotal Failures: " + failures + "\t" + ul + "(" + ul.getDisplayName(ULocale.ENGLISH)
215                                + ")");
216                        failures = 0;
217                    }
218                }
219            } catch (Exception e) {
220                StringWriter sw = new StringWriter();
221                PrintWriter pw = new PrintWriter(sw);
222                e.printStackTrace(pw);
223                pw.flush();
224                errln("Exception: Locale: " + ul + ",\tValue: <" + value + ">\r\n" + sw.toString());
225            }
226        }
227
228        public void loglnSAX(String message) {
229            String temp = message + "\t[" + name;
230            for (Iterator it = settings.keySet().iterator(); it.hasNext();) {
231                String attributeName = (String) it.next();
232                String attributeValue = (String) settings.get(attributeName);
233                temp += " " + attributeName + "=<" + attributeValue + ">";
234            }
235            logln(temp + "]");
236        }
237
238        int lookupValue(Object x, Object[] list) {
239            for (int i = 0; i < list.length; ++i) {
240                if (x.equals(list[i]))
241                    return i;
242            }
243            loglnSAX("Unknown String: " + x);
244            return -1;
245        }
246
247        abstract void handleResult(ULocale currentLocale, String value) throws Exception;
248
249        /**
250         * @param attributes
251         */
252        public void setAttributes(Attributes attributes) {
253            String localeList = attributes.getValue("locales");
254            String[] currentLocaleString = new String[50];
255            android.icu.impl.Utility.split(localeList, ' ', currentLocaleString);
256            currentLocales.clear();
257            for (int i = 0; i < currentLocaleString.length; ++i) {
258                if (currentLocaleString[i].length() == 0)
259                    continue;
260                if (allLocales.contains("")) {
261                    logln("Skipping locale, not in ICU4J: " + currentLocaleString[i]);
262                    continue;
263                }
264                currentLocales.add(new ULocale(currentLocaleString[i]));
265            }
266            if (DEBUG)
267                logln("Setting locales: " + currentLocales);
268        }
269    }
270
271    public Handler getHandler(String name, Attributes attributes) {
272        if (DEBUG)
273            logln("Creating Handler: " + name);
274        Handler result = (Handler) RegisteredHandlers.get(name);
275        if (result == null)
276            logln("Unexpected test type: " + name);
277        else {
278            result.setAttributes(attributes);
279        }
280        return result;
281    }
282
283    public void addHandler(String name, Handler handler) {
284        if (!TEST_MATCH.reset(name).matches())
285            handler = new NullHandler();
286        handler.setName(name);
287        RegisteredHandlers.put(name, handler);
288    }
289
290    Map RegisteredHandlers = new HashMap();
291
292    class NullHandler extends Handler {
293        void handleResult(ULocale currentLocale, String value) throws Exception {
294        }
295    }
296
297    // ============ Statics for Date/Number Support ============
298
299    static TimeZone utc = TimeZone.getTimeZone("GMT");
300    static DateFormat iso = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'");
301    {
302        iso.setTimeZone(utc);
303    }
304
305    static int[] DateFormatValues = { -1, DateFormat.SHORT, DateFormat.MEDIUM, DateFormat.LONG, DateFormat.FULL };
306
307    // The following are different data format types that are part of the parameters in CLDR
308    static String[] DateFormatNames = { "none", "short", "medium", "long", "full" };
309
310    // The following are different number types that are part of the parameters in CLDR
311    static String[] NumberNames = { "standard", "integer", "decimal", "percent", "scientific", "GBP" };
312
313
314    // ============ Handler for Collation ============
315    static UnicodeSet controlsAndSpace = new UnicodeSet("[:cc:]");
316
317    static String remove(String in, UnicodeSet toRemove) {
318        int cp;
319        StringBuffer result = new StringBuffer();
320        for (int i = 0; i < in.length(); i += UTF16.getCharCount(cp)) {
321            cp = UTF16.charAt(in, i);
322            if (!toRemove.contains(cp))
323                UTF16.append(result, cp);
324        }
325        return result.toString();
326    }
327
328    {
329        // johnvu: Collator was originally disabled
330        // TODO (dougfelt) move this test
331        /*
332          addHandler("collation", new Handler() {
333             public void handleResult(ULocale currentLocale, String value) {
334                 Collator col = Collator.getInstance(currentLocale);
335                 String lastLine = "";
336                 int count = 0;
337                 for (int pos = 0; pos < value.length();) {
338                     int nextPos = value.indexOf('\n', pos);
339                     if (nextPos < 0)
340                         nextPos = value.length();
341                     String line = value.substring(pos, nextPos);
342                     line = remove(line, controlsAndSpace);  HACK for SAX
343                     if (line.trim().length() != 0) {  HACK for SAX
344                         int comp = col.compare(lastLine, line);
345                         if (comp > 0) {
346                             failures++;
347                             errln("\tLine " + (count + 1) + "\tFailure: "
348                                     + showString(lastLine) + " should be leq "
349                                     + showString(line));
350                         } else if (DEBUG) {
351                             logln("OK: " + line);
352                         }
353                         lastLine = line;
354                     }
355                     pos = nextPos + 1;
356                     count++;
357                 }
358             }
359         });
360        */
361
362        // ============ Handler for Numbers ============
363        addHandler("number", new Handler() {
364            public void handleResult(ULocale locale, String result) {
365                NumberFormat nf = null;
366                double v = Double.NaN;
367                for (Iterator it = settings.keySet().iterator(); it.hasNext();) {
368                    String attributeName = (String) it.next();
369                    String attributeValue = (String) settings.get(attributeName);
370
371                    // Checks if the attribute name is a draft and whether
372                    // or not it has been approved / contributed by CLDR yet
373                    // otherwise, skips it because it is most likely rejected by ICU
374                    if (attributeName.equals("draft")) {
375                        if (attributeValue.indexOf("approved") == -1 && attributeValue.indexOf("contributed") == -1) {
376                            break;
377                        }
378                        continue;
379                    }
380
381                    // Update the value to be checked
382                    if (attributeName.equals("input")) {
383                        v = Double.parseDouble(attributeValue);
384                        continue;
385                    }
386
387                    // At this point, it must be a numberType
388                    int index = lookupValue(attributeValue, NumberNames);
389
390                    if (DEBUG)
391                        logln("Getting number format for " + locale);
392                    switch (index) {
393                    case 0:
394                        nf = NumberFormat.getInstance(locale);
395                        break;
396                    case 1:
397                        nf = NumberFormat.getIntegerInstance(locale);
398                        break;
399                    case 2:
400                        nf = NumberFormat.getNumberInstance(locale);
401                        break;
402                    case 3:
403                        nf = NumberFormat.getPercentInstance(locale);
404                        break;
405                    case 4:
406                        nf = NumberFormat.getScientificInstance(locale);
407                        break;
408                    default:
409                        nf = NumberFormat.getCurrencyInstance(locale);
410                        nf.setCurrency(Currency.getInstance(attributeValue));
411                        break;
412                    }
413                    String temp = nf.format(v).trim();
414                    result = result.trim(); // HACK because of SAX
415                    if (!temp.equals(result)) {
416                        logln("Number: Locale: " + locale +
417                                "\n\tType: " + attributeValue +
418                                "\n\tDraft: " + settings.get("draft") +
419                                "\n\tCLDR: <" + result + ">" +
420                                "\n\tICU: <" + temp + ">");
421                    }
422
423                }
424            }
425        });
426
427        // ============ Handler for Dates ============
428        addHandler("date", new Handler() {
429            public void handleResult(ULocale locale, String result) throws ParseException {
430                int dateFormat = 0;
431                int timeFormat = 0;
432                Date date = new Date();
433                boolean approved = true;
434
435                for (Iterator it = settings.keySet().iterator(); it.hasNext();) {
436                    String attributeName = (String) it.next();
437                    String attributeValue = (String) settings.get(attributeName);
438
439                    // Checks if the attribute name is a draft and whether
440                    // or not it has been approved / contributed by CLDR yet
441                    // otherwise, skips it because it is most likely rejected by ICU
442                    if (attributeName.equals("draft")) {
443                        if (attributeValue.indexOf("approved") == -1 && attributeValue.indexOf("contributed") == -1) {
444                            approved = false;
445                            break;
446                        }
447                        continue;
448                    }
449
450                    // Update the value to be checked
451                    if (attributeName.equals("input")) {
452                        date = iso.parse(attributeValue);
453                        continue;
454                    }
455                    // At this point, it must be either dateType or timeType
456                    int index = lookupValue(attributeValue, DateFormatNames);
457                    if (attributeName.equals("dateType"))
458                        dateFormat = index;
459                    else if (attributeName.equals("timeType"))
460                        timeFormat = index;
461
462                }
463
464                // The attribute value must be approved in order to be checked,
465                // if it hasn't been approved, it shouldn't be checked if it
466                // matches with ICU
467                if (approved) {
468                    SimpleDateFormat dt = getDateFormat(locale, dateFormat, timeFormat);
469                    dt.setTimeZone(utc);
470                    String temp = dt.format(date).trim();
471                    result = result.trim(); // HACK because of SAX
472                    if (!temp.equals(result)) {
473                        logln("DateTime: Locale: " + locale +
474                                "\n\tDate: " + DateFormatNames[dateFormat] +
475                                "\n\tTime: " + DateFormatNames[timeFormat] +
476                                "\n\tDraft: " + settings.get("draft") +
477                                "\n\tCLDR: <" + result + "> " +
478                                "\n\tICU: <" + temp + ">");
479                    }
480                }
481            }
482
483            private SimpleDateFormat getDateFormat(ULocale locale, int dateFormat, int timeFormat) {
484                if (DEBUG)
485                    logln("Getting date/time format for " + locale);
486                if (DEBUG && "ar_EG".equals(locale.toString())) {
487                    logln("debug here");
488                }
489                DateFormat dt;
490                if (dateFormat == 0) {
491                    dt = DateFormat.getTimeInstance(DateFormatValues[timeFormat], locale);
492                    if (DEBUG)
493                        System.out.print("getTimeInstance");
494                } else if (timeFormat == 0) {
495                    dt = DateFormat.getDateInstance(DateFormatValues[dateFormat], locale);
496                    if (DEBUG)
497                        System.out.print("getDateInstance");
498                } else {
499                    dt = DateFormat.getDateTimeInstance(DateFormatValues[dateFormat], DateFormatValues[timeFormat],
500                            locale);
501                    if (DEBUG)
502                        System.out.print("getDateTimeInstance");
503                }
504                if (DEBUG)
505                    logln("\tinput:\t" + dateFormat + ", " + timeFormat + " => " + ((SimpleDateFormat) dt).toPattern());
506                return (SimpleDateFormat) dt;
507            }
508        });
509
510        // ============ Handler for Zones ============
511        addHandler("zoneFields", new Handler() {
512            String date = "";
513            String zone = "";
514            String parse = "";
515            String pattern = "";
516
517            public void handleResult(ULocale locale, String result) throws ParseException {
518                for (Iterator it = settings.keySet().iterator(); it.hasNext();) {
519                    String attributeName = (String) it.next();
520                    String attributeValue = (String) settings.get(attributeName);
521                    if (attributeName.equals("date")) {
522                        date = attributeValue;
523                    } else if (attributeName.equals("field")) {
524                        pattern = attributeValue;
525                    } else if (attributeName.equals("zone")) {
526                        zone = attributeValue;
527                    } else if (attributeName.equals("parse")) {
528                        parse = attributeValue;
529                    }
530                }
531
532                if (!ZONE_MATCH.reset(zone).matches()) return;
533                Date dateValue = iso.parse(date);
534                SimpleDateFormat field = new SimpleDateFormat(pattern, locale);
535                field.setTimeZone(TimeZone.getTimeZone(zone));
536                String temp = field.format(dateValue).trim();
537                // SKIP PARSE FOR NOW
538                result = result.trim(); // HACK because of SAX
539                if (!temp.equals(result)) {
540                    temp = field.format(dateValue).trim(); // call again for debugging
541                    logln("Zone Format: Locale: " + locale
542                            + "\n\tZone: " + zone
543                            + "\n\tDate: " + date
544                            + "\n\tField: " + pattern
545                            + "\n\tParse: " + parse
546                            + "\n\tDraft: " + settings.get("draft")
547                            + "\n\tCLDR: <" + result
548                            + ">\n\tICU: <" + temp + ">");
549                }
550            }
551        });
552    }
553
554    // ============ Gorp for SAX ============
555
556    {
557        try {
558            SAXParserFactory factory = SAXParserFactory.newInstance();
559            factory.setValidating(true);
560            SAX = factory.newSAXParser();
561        } catch (Exception e) {
562            throw new IllegalArgumentException("SAXParserFacotry was unable to start.");
563        }
564    }
565
566    DefaultHandler DEFAULT_HANDLER = new DefaultHandler() {
567        static final boolean DEBUG = false;
568        StringBuffer lastChars = new StringBuffer();
569        // boolean justPopped = false;
570        Handler handler;
571
572        public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
573            // data.put(new ContextStack(contextStack), lastChars);
574            // lastChars = "";
575            try {
576                if (qName.equals("cldrTest")) {
577                    // skip
578                } else if (qName.equals("result") && handler != null) {
579                    for (int i = 0; i < attributes.getLength(); ++i) {
580                        handler.set(attributes.getQName(i), attributes.getValue(i));
581                    }
582                } else {
583                    handler = getHandler(qName, attributes);
584                    // handler.set("locale", uLocale.toString());
585                }
586                // if (DEBUG) logln("startElement:\t" + contextStack);
587                // justPopped = false;
588            } catch (RuntimeException e) {
589                e.printStackTrace();
590                throw e;
591            }
592        }
593
594        public void endElement(String uri, String localName, String qName) throws SAXException {
595            try {
596                // if (DEBUG) logln("endElement:\t" + contextStack);
597                if (qName.equals("result") && handler != null) {
598                    handler.checkResult(lastChars.toString());
599                } else if (qName.length() != 0) {
600                    // logln("Unexpected contents of: " + qName + ", <" + lastChars + ">");
601                }
602                lastChars.setLength(0);
603                // justPopped = true;
604            } catch (RuntimeException e) {
605                e.printStackTrace();
606                throw e;
607            }
608        }
609
610        // Have to hack around the fact that the character data might be in pieces
611        public void characters(char[] ch, int start, int length) throws SAXException {
612            try {
613                String value = new String(ch, start, length);
614                if (DEBUG)
615                    logln("characters:\t" + value);
616                lastChars.append(value);
617                // justPopped = false;
618            } catch (RuntimeException e) {
619                e.printStackTrace();
620                throw e;
621            }
622        }
623
624        // just for debugging
625
626        public void notationDecl(String name, String publicId, String systemId) throws SAXException {
627            logln("notationDecl: " + name + ", " + publicId + ", " + systemId);
628        }
629
630        public void processingInstruction(String target, String data) throws SAXException {
631            logln("processingInstruction: " + target + ", " + data);
632        }
633
634        public void skippedEntity(String name) throws SAXException {
635            logln("skippedEntity: " + name);
636        }
637
638        public void unparsedEntityDecl(String name, String publicId, String systemId, String notationName)
639                throws SAXException {
640            logln("unparsedEntityDecl: " + name + ", " + publicId + ", " + systemId + ", " + notationName);
641        }
642    };
643}
644