1/* GENERATED SOURCE. DO NOT MODIFY. */ 2// © 2016 and later: Unicode, Inc. and others. 3// License & terms of use: http://www.unicode.org/copyright.html#License 4/* 5********************************************************************** 6* Copyright (c) 2002-2010, International Business Machines 7* Corporation and others. All Rights Reserved. 8********************************************************************** 9* Author: Mark Davis 10********************************************************************** 11*/ 12package android.icu.dev.test.cldr; 13 14import java.io.File; 15import java.io.IOException; 16import java.io.PrintWriter; 17import java.io.StringWriter; 18import java.text.ParseException; 19import java.util.ArrayList; 20import java.util.Collection; 21import java.util.Date; 22import java.util.HashMap; 23import java.util.Iterator; 24import java.util.List; 25import java.util.Map; 26import java.util.Set; 27import java.util.TreeMap; 28import java.util.TreeSet; 29import java.util.regex.Matcher; 30import java.util.regex.Pattern; 31 32import javax.xml.parsers.SAXParser; 33import javax.xml.parsers.SAXParserFactory; 34 35import org.junit.Ignore; 36import org.junit.Test; 37import org.xml.sax.Attributes; 38import org.xml.sax.SAXException; 39import org.xml.sax.helpers.DefaultHandler; 40 41import android.icu.dev.test.TestFmwk; 42import android.icu.text.DateFormat; 43import android.icu.text.NumberFormat; 44import android.icu.text.SimpleDateFormat; 45import android.icu.text.UTF16; 46import android.icu.text.UnicodeSet; 47import android.icu.util.Currency; 48import android.icu.util.TimeZone; 49import android.icu.util.ULocale; 50 51/** 52 * This is a test file that takes in the CLDR XML test files and test against 53 * ICU4J. This test file is used to verify that ICU4J is implemented correctly. 54 * As it stands, the test generates all the errors to the console by logging it. 55 * The logging is only possible if "-v" or verbose is set as an argument. 56 * This will allow users to know what problems occurred within CLDR and ICU. 57 * Collator was disabled in this test file and therefore will be skipped. 58 * 59 * Instructions: 60 * 1) In order for this to work correctly, you must download the latest CLDR data 61 * in the form of XML. You must also set the CLDR directory using: 62 * -DCLDR_DIRECTORY=<top level of cldr> 63 * 2) You may also consider increasing the memory using -Xmx512m. 64 * 3) For speed purposes, you may consider creating a temporary directory for the 65 * CLDR cache using: 66 * -DCLDR_DTD_CACHE=<cldr cache directory> 67 * 4) You may use other environment variables to narrow down your tests using: 68 * -DXML_MATCH=".*" 69 * -DXML_MATCH="de.*" (or whatever regex you want) to just test certain locales. 70 * -DTEST_MATCH="zone.*" (or whatever regex you want) to just test collation, numbers, etc. 71 * -DZONE_MATCH="(?!America/Argentina).*" 72 * -DZONE_MATCH=".*Moscow.*" (to only test certain zones) 73 74 * @author medavis 75 * @author John Huan Vu (johnvu@us.ibm.com) 76 */ 77public class TestCLDRVsICU extends TestFmwk { 78 static final boolean DEBUG = false; 79 80 // ULocale uLocale = ULocale.ENGLISH; 81 // Locale oLocale = Locale.ENGLISH; // TODO Drop once ICU4J has ULocale everywhere 82 // static PrintWriter log; 83 SAXParser SAX; 84 static Matcher LOCALE_MATCH, TEST_MATCH, ZONE_MATCH; 85 static String CLDR_DIRECTORY; 86 static { 87 System.out.println(); 88 LOCALE_MATCH = getEnvironmentRegex("XML_MATCH", ".*"); 89 TEST_MATCH = getEnvironmentRegex("TEST_MATCH", ".*"); 90 ZONE_MATCH = getEnvironmentRegex("ZONE_MATCH", ".*"); 91 92 // CLDR_DIRECTORY is where all the CLDR XML test files are located 93 // WARNING: THIS IS TEMPORARY DIRECTORY UNTIL THE FILES ARE STRAIGHTENED OUT 94 CLDR_DIRECTORY = getEnvironmentString("CLDR_DIRECTORY", "C:\\Unicode-CVS2\\cldr\\"); 95 System.out.println(); 96 } 97 98 private static Matcher getEnvironmentRegex(String key, String defaultValue) { 99 return Pattern.compile(getEnvironmentString(key, defaultValue)).matcher(""); 100 } 101 102 private static String getEnvironmentString(String key, String defaultValue) { 103 String temp = System.getProperty(key); 104 if (temp == null) 105 temp = defaultValue; 106 else 107 System.out.print("-D" + key + "=\"" + temp + "\" "); 108 return temp; 109 } 110 111 Set allLocales = new TreeSet(); 112 113 // TODO(junit): seems to be failing with missing locales - maybe rewrite as parameterized 114 @Ignore 115 @Test 116 public void TestFiles() throws SAXException, IOException { 117 // only get ICU's locales 118 Set s = new TreeSet(); 119 addLocales(NumberFormat.getAvailableULocales(), s); 120 addLocales(DateFormat.getAvailableULocales(), s); 121 122 // johnvu: Collator was originally disabled 123 // addLocales(Collator.getAvailableULocales(), s); 124 125 // filter, to make tracking down bugs easier 126 for (Iterator it = s.iterator(); it.hasNext();) { 127 String locale = (String) it.next(); 128 if (!LOCALE_MATCH.reset(locale).matches()) 129 continue; 130 _test(locale); 131 } 132 } 133 134 public void addLocales(ULocale[] list, Collection s) { 135 for (int i = 0; i < list.length; ++i) { 136 allLocales.add(list[i].toString()); 137 s.add(list[i].getLanguage()); 138 } 139 } 140 141 public String getLanguage(ULocale uLocale) { 142 String result = uLocale.getLanguage(); 143 String script = uLocale.getScript(); 144 if (script.length() != 0) 145 result += "_" + script; 146 return result; 147 } 148 149 private void _test(String localeName) throws SAXException, IOException { 150 // uLocale = new ULocale(localeName); 151 // oLocale = uLocale.toLocale(); 152 153 File f = new File(CLDR_DIRECTORY, "test/" + localeName + ".xml"); 154 logln("Testing " + f.getCanonicalPath()); 155 SAX.parse(f, DEFAULT_HANDLER); 156 } 157 158 private static class ToHex { 159 public String transliterate(String in) { 160 StringBuilder sb = new StringBuilder(); 161 for (int i = 0; i < in.length(); ++i) { 162 char c = in.charAt(i); 163 sb.append("\\u"); 164 if (c < 1000) { 165 sb.append('0'); 166 if (c < 100) { 167 sb.append('0'); 168 if (c < 10) { 169 sb.append('0'); 170 } 171 } 172 } 173 sb.append(Integer.toHexString((int) c)); 174 } 175 return sb.toString(); 176 } 177 } 178 179 // static Transliterator toUnicode = Transliterator.getInstance("any-hex"); 180 private static final ToHex toUnicode = new ToHex(); 181 182 static public String showString(String in) { 183 return "\u00AB" + in + "\u00BB (" + toUnicode.transliterate(in) + ")"; 184 } 185 186 // ============ SAX Handler Infrastructure ============ 187 188 abstract public class Handler { 189 Map settings = new TreeMap(); 190 String name; 191 List currentLocales = new ArrayList(); 192 int failures = 0; 193 194 void setName(String name) { 195 this.name = name; 196 } 197 198 void set(String attributeName, String attributeValue) { 199 // if (DEBUG) logln(attributeName + " => " + attributeValue); 200 settings.put(attributeName, attributeValue); 201 } 202 203 void checkResult(String value) { 204 if (settings.get("draft").equals("unconfirmed") || settings.get("draft").equals("provisional")) { 205 return; // skip draft 206 } 207 ULocale ul = new ULocale("xx"); 208 try { 209 for (int i = 0; i < currentLocales.size(); ++i) { 210 ul = (ULocale) currentLocales.get(i); 211 // loglnSAX(" Checking " + ul + "(" + ul.getDisplayName(ULocale.ENGLISH) + ")" + " for " + name); 212 handleResult(ul, value); 213 if (failures != 0) { 214 errln("\tTotal Failures: " + failures + "\t" + ul + "(" + ul.getDisplayName(ULocale.ENGLISH) 215 + ")"); 216 failures = 0; 217 } 218 } 219 } catch (Exception e) { 220 StringWriter sw = new StringWriter(); 221 PrintWriter pw = new PrintWriter(sw); 222 e.printStackTrace(pw); 223 pw.flush(); 224 errln("Exception: Locale: " + ul + ",\tValue: <" + value + ">\r\n" + sw.toString()); 225 } 226 } 227 228 public void loglnSAX(String message) { 229 String temp = message + "\t[" + name; 230 for (Iterator it = settings.keySet().iterator(); it.hasNext();) { 231 String attributeName = (String) it.next(); 232 String attributeValue = (String) settings.get(attributeName); 233 temp += " " + attributeName + "=<" + attributeValue + ">"; 234 } 235 logln(temp + "]"); 236 } 237 238 int lookupValue(Object x, Object[] list) { 239 for (int i = 0; i < list.length; ++i) { 240 if (x.equals(list[i])) 241 return i; 242 } 243 loglnSAX("Unknown String: " + x); 244 return -1; 245 } 246 247 abstract void handleResult(ULocale currentLocale, String value) throws Exception; 248 249 /** 250 * @param attributes 251 */ 252 public void setAttributes(Attributes attributes) { 253 String localeList = attributes.getValue("locales"); 254 String[] currentLocaleString = new String[50]; 255 android.icu.impl.Utility.split(localeList, ' ', currentLocaleString); 256 currentLocales.clear(); 257 for (int i = 0; i < currentLocaleString.length; ++i) { 258 if (currentLocaleString[i].length() == 0) 259 continue; 260 if (allLocales.contains("")) { 261 logln("Skipping locale, not in ICU4J: " + currentLocaleString[i]); 262 continue; 263 } 264 currentLocales.add(new ULocale(currentLocaleString[i])); 265 } 266 if (DEBUG) 267 logln("Setting locales: " + currentLocales); 268 } 269 } 270 271 public Handler getHandler(String name, Attributes attributes) { 272 if (DEBUG) 273 logln("Creating Handler: " + name); 274 Handler result = (Handler) RegisteredHandlers.get(name); 275 if (result == null) 276 logln("Unexpected test type: " + name); 277 else { 278 result.setAttributes(attributes); 279 } 280 return result; 281 } 282 283 public void addHandler(String name, Handler handler) { 284 if (!TEST_MATCH.reset(name).matches()) 285 handler = new NullHandler(); 286 handler.setName(name); 287 RegisteredHandlers.put(name, handler); 288 } 289 290 Map RegisteredHandlers = new HashMap(); 291 292 class NullHandler extends Handler { 293 void handleResult(ULocale currentLocale, String value) throws Exception { 294 } 295 } 296 297 // ============ Statics for Date/Number Support ============ 298 299 static TimeZone utc = TimeZone.getTimeZone("GMT"); 300 static DateFormat iso = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss'Z'"); 301 { 302 iso.setTimeZone(utc); 303 } 304 305 static int[] DateFormatValues = { -1, DateFormat.SHORT, DateFormat.MEDIUM, DateFormat.LONG, DateFormat.FULL }; 306 307 // The following are different data format types that are part of the parameters in CLDR 308 static String[] DateFormatNames = { "none", "short", "medium", "long", "full" }; 309 310 // The following are different number types that are part of the parameters in CLDR 311 static String[] NumberNames = { "standard", "integer", "decimal", "percent", "scientific", "GBP" }; 312 313 314 // ============ Handler for Collation ============ 315 static UnicodeSet controlsAndSpace = new UnicodeSet("[:cc:]"); 316 317 static String remove(String in, UnicodeSet toRemove) { 318 int cp; 319 StringBuffer result = new StringBuffer(); 320 for (int i = 0; i < in.length(); i += UTF16.getCharCount(cp)) { 321 cp = UTF16.charAt(in, i); 322 if (!toRemove.contains(cp)) 323 UTF16.append(result, cp); 324 } 325 return result.toString(); 326 } 327 328 { 329 // johnvu: Collator was originally disabled 330 // TODO (dougfelt) move this test 331 /* 332 addHandler("collation", new Handler() { 333 public void handleResult(ULocale currentLocale, String value) { 334 Collator col = Collator.getInstance(currentLocale); 335 String lastLine = ""; 336 int count = 0; 337 for (int pos = 0; pos < value.length();) { 338 int nextPos = value.indexOf('\n', pos); 339 if (nextPos < 0) 340 nextPos = value.length(); 341 String line = value.substring(pos, nextPos); 342 line = remove(line, controlsAndSpace); HACK for SAX 343 if (line.trim().length() != 0) { HACK for SAX 344 int comp = col.compare(lastLine, line); 345 if (comp > 0) { 346 failures++; 347 errln("\tLine " + (count + 1) + "\tFailure: " 348 + showString(lastLine) + " should be leq " 349 + showString(line)); 350 } else if (DEBUG) { 351 logln("OK: " + line); 352 } 353 lastLine = line; 354 } 355 pos = nextPos + 1; 356 count++; 357 } 358 } 359 }); 360 */ 361 362 // ============ Handler for Numbers ============ 363 addHandler("number", new Handler() { 364 public void handleResult(ULocale locale, String result) { 365 NumberFormat nf = null; 366 double v = Double.NaN; 367 for (Iterator it = settings.keySet().iterator(); it.hasNext();) { 368 String attributeName = (String) it.next(); 369 String attributeValue = (String) settings.get(attributeName); 370 371 // Checks if the attribute name is a draft and whether 372 // or not it has been approved / contributed by CLDR yet 373 // otherwise, skips it because it is most likely rejected by ICU 374 if (attributeName.equals("draft")) { 375 if (attributeValue.indexOf("approved") == -1 && attributeValue.indexOf("contributed") == -1) { 376 break; 377 } 378 continue; 379 } 380 381 // Update the value to be checked 382 if (attributeName.equals("input")) { 383 v = Double.parseDouble(attributeValue); 384 continue; 385 } 386 387 // At this point, it must be a numberType 388 int index = lookupValue(attributeValue, NumberNames); 389 390 if (DEBUG) 391 logln("Getting number format for " + locale); 392 switch (index) { 393 case 0: 394 nf = NumberFormat.getInstance(locale); 395 break; 396 case 1: 397 nf = NumberFormat.getIntegerInstance(locale); 398 break; 399 case 2: 400 nf = NumberFormat.getNumberInstance(locale); 401 break; 402 case 3: 403 nf = NumberFormat.getPercentInstance(locale); 404 break; 405 case 4: 406 nf = NumberFormat.getScientificInstance(locale); 407 break; 408 default: 409 nf = NumberFormat.getCurrencyInstance(locale); 410 nf.setCurrency(Currency.getInstance(attributeValue)); 411 break; 412 } 413 String temp = nf.format(v).trim(); 414 result = result.trim(); // HACK because of SAX 415 if (!temp.equals(result)) { 416 logln("Number: Locale: " + locale + 417 "\n\tType: " + attributeValue + 418 "\n\tDraft: " + settings.get("draft") + 419 "\n\tCLDR: <" + result + ">" + 420 "\n\tICU: <" + temp + ">"); 421 } 422 423 } 424 } 425 }); 426 427 // ============ Handler for Dates ============ 428 addHandler("date", new Handler() { 429 public void handleResult(ULocale locale, String result) throws ParseException { 430 int dateFormat = 0; 431 int timeFormat = 0; 432 Date date = new Date(); 433 boolean approved = true; 434 435 for (Iterator it = settings.keySet().iterator(); it.hasNext();) { 436 String attributeName = (String) it.next(); 437 String attributeValue = (String) settings.get(attributeName); 438 439 // Checks if the attribute name is a draft and whether 440 // or not it has been approved / contributed by CLDR yet 441 // otherwise, skips it because it is most likely rejected by ICU 442 if (attributeName.equals("draft")) { 443 if (attributeValue.indexOf("approved") == -1 && attributeValue.indexOf("contributed") == -1) { 444 approved = false; 445 break; 446 } 447 continue; 448 } 449 450 // Update the value to be checked 451 if (attributeName.equals("input")) { 452 date = iso.parse(attributeValue); 453 continue; 454 } 455 // At this point, it must be either dateType or timeType 456 int index = lookupValue(attributeValue, DateFormatNames); 457 if (attributeName.equals("dateType")) 458 dateFormat = index; 459 else if (attributeName.equals("timeType")) 460 timeFormat = index; 461 462 } 463 464 // The attribute value must be approved in order to be checked, 465 // if it hasn't been approved, it shouldn't be checked if it 466 // matches with ICU 467 if (approved) { 468 SimpleDateFormat dt = getDateFormat(locale, dateFormat, timeFormat); 469 dt.setTimeZone(utc); 470 String temp = dt.format(date).trim(); 471 result = result.trim(); // HACK because of SAX 472 if (!temp.equals(result)) { 473 logln("DateTime: Locale: " + locale + 474 "\n\tDate: " + DateFormatNames[dateFormat] + 475 "\n\tTime: " + DateFormatNames[timeFormat] + 476 "\n\tDraft: " + settings.get("draft") + 477 "\n\tCLDR: <" + result + "> " + 478 "\n\tICU: <" + temp + ">"); 479 } 480 } 481 } 482 483 private SimpleDateFormat getDateFormat(ULocale locale, int dateFormat, int timeFormat) { 484 if (DEBUG) 485 logln("Getting date/time format for " + locale); 486 if (DEBUG && "ar_EG".equals(locale.toString())) { 487 logln("debug here"); 488 } 489 DateFormat dt; 490 if (dateFormat == 0) { 491 dt = DateFormat.getTimeInstance(DateFormatValues[timeFormat], locale); 492 if (DEBUG) 493 System.out.print("getTimeInstance"); 494 } else if (timeFormat == 0) { 495 dt = DateFormat.getDateInstance(DateFormatValues[dateFormat], locale); 496 if (DEBUG) 497 System.out.print("getDateInstance"); 498 } else { 499 dt = DateFormat.getDateTimeInstance(DateFormatValues[dateFormat], DateFormatValues[timeFormat], 500 locale); 501 if (DEBUG) 502 System.out.print("getDateTimeInstance"); 503 } 504 if (DEBUG) 505 logln("\tinput:\t" + dateFormat + ", " + timeFormat + " => " + ((SimpleDateFormat) dt).toPattern()); 506 return (SimpleDateFormat) dt; 507 } 508 }); 509 510 // ============ Handler for Zones ============ 511 addHandler("zoneFields", new Handler() { 512 String date = ""; 513 String zone = ""; 514 String parse = ""; 515 String pattern = ""; 516 517 public void handleResult(ULocale locale, String result) throws ParseException { 518 for (Iterator it = settings.keySet().iterator(); it.hasNext();) { 519 String attributeName = (String) it.next(); 520 String attributeValue = (String) settings.get(attributeName); 521 if (attributeName.equals("date")) { 522 date = attributeValue; 523 } else if (attributeName.equals("field")) { 524 pattern = attributeValue; 525 } else if (attributeName.equals("zone")) { 526 zone = attributeValue; 527 } else if (attributeName.equals("parse")) { 528 parse = attributeValue; 529 } 530 } 531 532 if (!ZONE_MATCH.reset(zone).matches()) return; 533 Date dateValue = iso.parse(date); 534 SimpleDateFormat field = new SimpleDateFormat(pattern, locale); 535 field.setTimeZone(TimeZone.getTimeZone(zone)); 536 String temp = field.format(dateValue).trim(); 537 // SKIP PARSE FOR NOW 538 result = result.trim(); // HACK because of SAX 539 if (!temp.equals(result)) { 540 temp = field.format(dateValue).trim(); // call again for debugging 541 logln("Zone Format: Locale: " + locale 542 + "\n\tZone: " + zone 543 + "\n\tDate: " + date 544 + "\n\tField: " + pattern 545 + "\n\tParse: " + parse 546 + "\n\tDraft: " + settings.get("draft") 547 + "\n\tCLDR: <" + result 548 + ">\n\tICU: <" + temp + ">"); 549 } 550 } 551 }); 552 } 553 554 // ============ Gorp for SAX ============ 555 556 { 557 try { 558 SAXParserFactory factory = SAXParserFactory.newInstance(); 559 factory.setValidating(true); 560 SAX = factory.newSAXParser(); 561 } catch (Exception e) { 562 throw new IllegalArgumentException("SAXParserFacotry was unable to start."); 563 } 564 } 565 566 DefaultHandler DEFAULT_HANDLER = new DefaultHandler() { 567 static final boolean DEBUG = false; 568 StringBuffer lastChars = new StringBuffer(); 569 // boolean justPopped = false; 570 Handler handler; 571 572 public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException { 573 // data.put(new ContextStack(contextStack), lastChars); 574 // lastChars = ""; 575 try { 576 if (qName.equals("cldrTest")) { 577 // skip 578 } else if (qName.equals("result") && handler != null) { 579 for (int i = 0; i < attributes.getLength(); ++i) { 580 handler.set(attributes.getQName(i), attributes.getValue(i)); 581 } 582 } else { 583 handler = getHandler(qName, attributes); 584 // handler.set("locale", uLocale.toString()); 585 } 586 // if (DEBUG) logln("startElement:\t" + contextStack); 587 // justPopped = false; 588 } catch (RuntimeException e) { 589 e.printStackTrace(); 590 throw e; 591 } 592 } 593 594 public void endElement(String uri, String localName, String qName) throws SAXException { 595 try { 596 // if (DEBUG) logln("endElement:\t" + contextStack); 597 if (qName.equals("result") && handler != null) { 598 handler.checkResult(lastChars.toString()); 599 } else if (qName.length() != 0) { 600 // logln("Unexpected contents of: " + qName + ", <" + lastChars + ">"); 601 } 602 lastChars.setLength(0); 603 // justPopped = true; 604 } catch (RuntimeException e) { 605 e.printStackTrace(); 606 throw e; 607 } 608 } 609 610 // Have to hack around the fact that the character data might be in pieces 611 public void characters(char[] ch, int start, int length) throws SAXException { 612 try { 613 String value = new String(ch, start, length); 614 if (DEBUG) 615 logln("characters:\t" + value); 616 lastChars.append(value); 617 // justPopped = false; 618 } catch (RuntimeException e) { 619 e.printStackTrace(); 620 throw e; 621 } 622 } 623 624 // just for debugging 625 626 public void notationDecl(String name, String publicId, String systemId) throws SAXException { 627 logln("notationDecl: " + name + ", " + publicId + ", " + systemId); 628 } 629 630 public void processingInstruction(String target, String data) throws SAXException { 631 logln("processingInstruction: " + target + ", " + data); 632 } 633 634 public void skippedEntity(String name) throws SAXException { 635 logln("skippedEntity: " + name); 636 } 637 638 public void unparsedEntityDecl(String name, String publicId, String systemId, String notationName) 639 throws SAXException { 640 logln("unparsedEntityDecl: " + name + ", " + publicId + ", " + systemId + ", " + notationName); 641 } 642 }; 643} 644