1//******************************************************************************* 2// 3// Copyright (C) 2003-2013, International Business Machines 4// Corporation and others. All Rights Reserved. 5// 6// file name: conversion.txt 7// encoding: US-ASCII 8// tab size: 8 (not used) 9// indentation:4 10// 11// created on: 2003jul15 12// created by: Markus W. Scherer 13// 14// ICU resource bundle source file with test data for data-driven conversion tests. 15// 16//******************************************************************************* 17 18conversion:table(nofallback) { 19 Info { 20 Description { "Test data for conversion" } 21 LongDescription { 22 "Test data for data-driven conversion tests in icu/source/test/intltest/convtest.cpp\n" 23 "Run intltest conversion\n" 24 25 "Charset names starting with '*' are for testdata names.\n" 26 "Charset names starting with '+' are for charsets currently not supported in ICU4J.\n" 27 28 "ICU callbacks are specified as strings with pairs of characters, each optional.\n" 29 "Callback function - '?'=Sub '0'=Skip '.'=Stop '&'=Escape\n" 30 "Callback option - a letter is passed in directly as const char * see ucnv_err.h\n" 31 "Empty string: Sub callback with NULL option\n" 32 33 "In order to specify a charset substitution character (for ucnv_setSubstChars()),\n" 34 "add a NUL (U+0000) to the callback string followed by the subchar bytes as Latin-1\n" 35 "characters. For example, for a Sub callback with no option and a subchar of FC FC,\n" 36 "use the string \"?\x00\xFC\xFC\"\n" 37 38 "In order to specify a substitution string (for ucnv_setSubstString()),\n" 39 "add an '=' to the callback string followed by the substitution string.\n" 40 "For example, for a Sub callback with no option and a substitution string\n" 41 "of \"ab\", use the string \"?=ab\"\n" 42 43 "fallbacks: per-direction boolean, currently only for fromUnicode; see Jitterbug 2401\n" 44 45 "errorCode: (empty)==zero | invalid | illegal | truncated | illesc | unsuppesc\n" 46 } 47 } 48 TestData { 49 toUnicode { 50 Headers { "charset", "bytes", "unicode", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidChars" } 51 Cases { 52 // Test ticket 6789: implement Java-compatible Unicode, UnicodeBig and UnicodeLittle converters 53 // For details about these encodings see convrtrs.txt. 54 // Standard UTF-16 55 { "UTF-16", :bin{ 00610062 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 56 { "UTF-16", :bin{ feff0061 }, "a", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 57 { "UTF-16", :bin{ fffe0061 }, "\u6100", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 58 // Java "Unicode" requires a BOM 59 { "+UTF-16,version=1", :bin{ 00610062 }, "\\x00\\x61b", :intvector{ 0,0,0,0,0,0,0,0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 60 { "+UTF-16,version=1", :bin{ feff0061 }, "a", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 61 { "+UTF-16,version=1", :bin{ fffe0061 }, "\u6100", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 62 // Standard UTF-16BE 63 { "UTF-16BE", :bin{ 00610062 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 64 { "UTF-16BE", :bin{ feff0061 }, "\ufeffa", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 65 { "UTF-16BE", :bin{ fffe0061 }, "\ufffea", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 66 // Java "UnicodeBig" requires a BE BOM or no BOM; it consumes the BE BOM 67 { "UTF-16BE,version=1",:bin{ 00610062 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 68 { "UnicodeBig", :bin{ feff0061 }, "a", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 69 { "UnicodeBig", :bin{ fffe0061 }, "\\xFF\\xFEa", :intvector{ 0,0,0,0,0,0,0,0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 70 // Standard UTF-16LE 71 { "UTF-16LE", :bin{ 61006200 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 72 { "UTF-16LE", :bin{ fffe6100 }, "\ufeffa", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 73 { "UTF-16LE", :bin{ feff6100 }, "\ufffea", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 74 // Java "UnicodeLittle" requires an LE BOM or no BOM; it consumes the LE BOM 75 { "UTF-16LE,version=1",:bin{ 61006200 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 76 { "UnicodeLittle", :bin{ fffe6100 }, "a", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 77 { "x-UTF-16LE-BOM", :bin{ feff6100 }, "\\xFE\\xFFa", :intvector{ 0,0,0,0,0,0,0,0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 78 79 // Test ticket 7704: implement Java-compatible "UTF-16" converter. 80 // Same as standard UTF-16 but fromUnicode always writes big-endian byte stream. 81 { "+UTF-16,version=2", :bin{ 00610062 }, "ab", :intvector{ 0,2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 82 { "+UTF-16,version=2", :bin{ feff0061 }, "a", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 83 { "+UTF-16,version=2", :bin{ fffe0061 }, "\u6100", :intvector{ 2 }, :int{1}, :int{0}, "", "&C", :bin{""} } 84 85 // Test ticket 5691: consistent illegal sequences 86 // The following test cases are for illegal character byte sequences. 87 // 88 // Unfortunately, we cannot use the Shift-JIS examples from the ticket 89 // comments because our Shift-JIS table is Windows-compatible and 90 // therefore has no illegal single bytes. Same for GBK. 91 // Instead, we use the stricter GB 18030 also for 2-byte examples. 92 // The byte sequences are generally slightly different from the ticket 93 // comment, simply using assigned characters rather than just 94 // theoretically valid sequences. 95 { 96 "gb18030", 97 :bin{ 618140813c81ff7a }, 98 "a\u4e02\\x81<\\x81\\xFFz", 99 :intvector{ 0,1,3,3,3,3,4,5,5,5,5,5,5,5,5,7 }, 100 :int{1}, :int{0}, "", "&C", :bin{""} 101 } 102 { 103 "EUC-JP", 104 :bin{ 618fb0a98fb03c8f3cb0a97a }, 105 "a\u4e28\\x8F\\xB0<\\x8F<\u9022z", 106 :intvector{ 0,1,4,4,4,4,5,5,5,5,6,7,7,7,7,8,9,11 }, 107 :int{1}, :int{0}, "", "&C", :bin{""} 108 } 109 { 110 "gb18030", 111 :bin{ 618130fc318130fc8181303c3e813cfc817afe90a8bc }, 112 "a\u05ed\\x810\u9f07\\x810<>\\x81<\u9f07z\ue854\u1e3f", 113 :intvector{ 0,1,5,5,5,5,6,7,9,9,9,9,10,11,12,13,13,13,13,14,15,17,18,20 }, 114 :int{1}, :int{0}, "", "&C", :bin{""} 115 } 116 { 117 "UTF-8", 118 :bin{ 61f1808182f180813cf18081fff180ff3cf1ff3c3e7a }, 119 "a\U00040042\\xF1\\x80\\x81<\\xF1\\x80\\x81\\xFF\\xF1\\x80\\xFF<\\xF1\\xFF<>z", 120 :intvector{ 0,1,1,5,5,5,5,5,5,5,5,5,5,5,5,8,9,9,9,9,9,9,9,9,9,9,9,9,12,12,12,12,13,13,13,13,13,13,13,13,15,15,15,15,16,17,17,17,17,18,18,18,18,19,20,21 }, 121 :int{1}, :int{0}, "", "&C", :bin{""} 122 } 123 { 124 "ISO-2022-JP", 125 :bin{ 1b24424141af4142affe41431b2842 }, 126 "\u758f\\xAF\u758e\\xAF\\xFE\u790e", 127 :intvector{ 3,5,5,5,5,6,8,8,8,8,8,8,8,8,10 }, 128 :int{1}, :int{0}, "", "&C", :bin{""} 129 } 130 { 131 "ibm-25546", 132 :bin{ 411b242943420e4141af4142affe41430f5a }, 133 "AB\uc88b\\xAF\uc88c\\xAF\\xFE\uc88dZ", 134 :intvector{ 0,5,7,9,9,9,9,10,12,12,12,12,12,12,12,12,14,17 }, 135 :int{1}, :int{0}, "", "&C", :bin{""} 136 } 137 { 138 "ISO-2022-KR", 139 :bin{ 411b242943420e4141af4142affe41430f5a }, 140 "AB\uc88b\\xAF\uc88c\\xAF\\xFE\uc88dZ", 141 :intvector{ 0,5,7,9,9,9,9,10,12,12,12,12,12,12,12,12,14,17 }, 142 :int{1}, :int{0}, "", "&C", :bin{""} 143 } 144 { 145 "ISO-2022-CN", 146 :bin{ 411b242941420e4141af4142affe41430f5a }, 147 "AB\u4eae\\xAF\u8c05\\xAF\\xFE\u64a9Z", 148 :intvector{ 0,5,7,9,9,9,9,10,12,12,12,12,12,12,12,12,14,17 }, 149 :int{1}, :int{0}, "", "&C", :bin{""} 150 } 151 { 152 "ISO-2022-CN-CNS", 153 :bin{ 411b2429470e21702541256f0f }, 154 "A\u00a7\u03c4\u02c7", 155 :intvector{ 0,6,8,10 }, 156 :int{1}, :int{0}, "", "&C", :bin{""} 157 } 158 { 159 "HZ", 160 :bin{ 417e7b4141af4142affe41437e7d5a }, 161 "A\u4eae\\xAF\u8c05\\xAF\\xFE\u64a9Z", 162 :intvector{ 0,3,5,5,5,5,6,8,8,8,8,8,8,8,8,10,14 }, 163 :int{1}, :int{0}, "", "&C", :bin{""} 164 } 165 // Test ticket 5691: consistent illegal sequences 166 // The following test cases are for illegal escape/designator/shift sequences. 167 // 168 // ISO-2022-JP and -CN with illegal escape sequences. 169 { 170 "ISO-2022-JP", 171 :bin{ 611b24201b244241411b283f1b28427a }, 172 "a\\x1B$ \u758f\\x1B\u2538z", 173 :intvector{ 0,1,1,1,1,2,3,7,9,9,9,9,10,15 }, 174 :int{1}, :int{0}, "", "&C", :bin{""} 175 } 176 { 177 "ISO-2022-CN", 178 :bin{ 611b2429201b2429410e41410f7a }, 179 "a\\x1B$) \u4eaez", 180 :intvector{ 0,1,1,1,1,2,3,4,10,13 }, 181 :int{1}, :int{0}, "", "&C", :bin{""} 182 } 183 // Test ticket 5691: ISO-2022-JP-2 with illegal single-shift SS2 and SS3 sequences. 184 // The first ESC N comes before its designator sequence, the last sequence is ESC+space. 185 { 186 "ISO-2022-JP-2", 187 :bin{ 4e1b4e4e1b2e414e1b4e4e4e1b204e }, 188 "N\\x1BNNN\xceN\\x1B N", 189 :intvector{ 0,1,1,1,1,2,3,7,10,11,12,12,12,12,13,14 }, 190 :int{1}, :int{0}, "", "&C", :bin{""} 191 } 192 { 193 "ISO-2022-CN-EXT", 194 :bin{ 4e1b4e4e1b242a484e1b4e4e4e4e1b204e }, 195 "N\\x1BNNN\u8f0eN\\x1B N", 196 :intvector{ 0,1,1,1,1,2,3,8,11,13,14,14,14,14,15,16 }, 197 :int{1}, :int{0}, "", "&C", :bin{""} 198 } 199 /* 200 * ICU 4.4 (ticket #7314) removes mappings for CNS 11643 planes 3..7 201 { 202 "ISO-2022-CN-EXT", 203 :bin{ 4f1b4f4f1b242b494f1b4f4f4f4f1b204f }, 204 "O\\x1BOOO\u492bO\\x1B O", 205 :intvector{ 0,1,1,1,1,2,3,8,11,13,14,14,14,14,15,16 }, 206 :int{1}, :int{0}, "", "&C", :bin{""} 207 } 208 */ 209 // Test ticket 5691: HZ with illegal tilde sequences. 210 { 211 "HZ", 212 :bin{ 417e20427e21437e80447e7b41417e207e41427e7f41437e7d5a }, 213 "A\\x7E B\\x7E!C\\x7E\\x80D\u4eae\\x7E\\x20\\x7E\u8c05\\x7E\\x7F\u64a9Z", 214 :intvector{ 0,1,1,1,1,2,3,4,4,4,4,5,6,7,7,7,7,7,7,7,7,9, // SBCS 215 12,14,14,14,14,14,14,14,14,16,16,16,16,17,19,19,19,19,19,19,19,19,21, // DBCS 216 25 }, // SBCS 217 :int{1}, :int{0}, "", "&C", :bin{""} 218 } 219 // Test ticket 5691: Example from Peter Edberg. 220 { 221 "ISO-2022-JP", 222 :bin{ 1b244230212f7e742630801b284a621b2458631b2842648061 }, 223 "\u4e9c\ufffd\u7199\ufffdb\ufffd$Xcd\ufffda", 224 :intvector{ 3,5,7,9,14,15,16,17,18,22,23,24 }, 225 :int{1}, :int{0}, "", "?", :bin{""} 226 } 227 // Test bug 6071 (2:1 Unicode:charset SBCS mapping). 228 { 229 "*test1bmp", 230 :bin{ 050008 }, 231 "e@uv", 232 :intvector{ 0,1,2,2 }, 233 :int{1}, :int{1}, "", "?", :bin{""} 234 } 235 // test that HZ limits its byte values to lead bytes 21..7d and trail bytes 21..7e 236 { 237 "HZ", 238 :bin{ 7e7b21212120217e217f772100007e217e7e7d207e7e807e0a2b }, 239 "\u3000\ufffd\u3013\ufffd\u9ccc\ufffd\ufffd\u3013 ~\ufffd+", 240 :intvector{ 2,4,6,8,10,12,14,15,19,20,22,25 }, 241 :int{1}, :int{1}, "", "?", :bin{""} 242 } 243 // improve coverage of ISO-2022-JP converter with hardcoded JIS X 0201 and 244 // using the Shift-JIS table for JIS X 0208 (ticket #5797) 245 { 246 "ISO-2022-JP", 247 :bin{ 1b284a7d7e801b2442306c20217f7e21202160217f22202225227f5f211b2842 }, 248 "}\u203e\ufffd\u4e00\ufffd\ufffd\ufffd\ufffd\xf7\ufffd\ufffd\u25b2\ufffd\u6f3e", 249 :intvector{ 3,4,5,9,11,12,14,16,17,19,21,23,25,27 }, 250 :int{1}, :int{1}, "", "?", :bin{""} 251 } 252 // improve coverage of ISO-2022-JP converter by simulating erroneous input 253 { 254 "ISO-2022-JP-2", 255 :bin{ 0f0ed11b2e41461b244141411b4e411b2e4147451b4ed31b2e4641411b4ed2 }, 256 "\u0046\u4eae\u00c1\u6865\u4eae", 257 :intvector{ 6, 10, 14, 18, 26 }, 258 :int{1}, :int{0}, "", "0", :bin{""} 259 } 260 // improve coverage of JIS7 converter by simulating incomplete shifted input 261 { 262 "JIS7", 263 :bin{ 0e11 }, 264 "", 265 :intvector{}, 266 :int{1}, :int{0}, "", "0", :bin{""} 267 } 268 // improve coverage of unrolled loops in ucnvmbcs.c/ucnv_MBCSSingleToBMPWithOffsets() 269 { 270 "ISO-8859-3", 271 :bin{ 000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f2021222324252627 }, 272 "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !\x22#$%&'", 273 :intvector{ 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39 }, 274 :int{1}, :int{0}, "", "?", :bin{""} 275 } 276 // test that ISO-2022-JP encodes ASCII as itself 277 { 278 "ISO-2022-JP", 279 :bin{ 3f4041424344454647 }, 280 "?@ABCDEFG", 281 :intvector{ 0,1,2,3,4,5,6,7,8 }, 282 :int{1}, :int{1}, "", "?", :bin{""} 283 } 284 // test that ISO-2022-CN encodes ASCII as itself 285 { 286 "ISO-2022-CN", 287 :bin{ 3f4041424344454647 }, 288 "?@ABCDEFG", 289 :intvector{ 0,1,2,3,4,5,6,7,8 }, 290 :int{1}, :int{1}, "", "?", :bin{""} 291 } 292 293 // ISO-2022-KR 294 295 // truncated, partial escape sequence 296 { 297 "ibm-25546", 298 :bin{ 1b }, "", :intvector{}, 299 :int{1}, :int{1}, "truncated", ".", :bin{ 1b } 300 } 301 { 302 "ibm-25546", 303 :bin{ 1b24 }, "", :intvector{}, 304 :int{1}, :int{1}, "truncated", ".", :bin{ 1b24 } 305 } 306 { 307 "ibm-25546", 308 :bin{ 1b2429 }, "", :intvector{}, 309 :int{1}, :int{1}, "truncated", ".", :bin{ 1b2429 } 310 } 311 // complete escape sequence but nothing else 312 { 313 "ibm-25546", 314 :bin{ 1b242943 }, "", :intvector{}, 315 :int{1}, :int{1}, "", ".", :bin{""} 316 } 317 { 318 "ibm-25546", 319 :bin{ 1b2429430e }, "", :intvector{}, 320 :int{1}, :int{1}, "", ".", :bin{""} 321 } 322 // escape plus ASCII character 323 { 324 "ibm-25546", 325 :bin{ 1b24294341 }, "A", :intvector{ 4 }, 326 :int{1}, :int{1}, "", ".", :bin{""} 327 } 328 // escape plus incomplete DBCS character 329 { 330 "ibm-25546", 331 :bin{ 1b2429430e41 }, "", :intvector{}, 332 :int{1}, :int{1}, "truncated", ".", :bin{ 41 } 333 } 334 // all complete with DBCS character 335 { 336 "ibm-25546", 337 :bin{ 1b2429430e4141 }, "\uc88b", :intvector{ 5 }, 338 :int{1}, :int{1}, "", ".", :bin{""} 339 } 340 // more complicated example 341 { 342 "ibm-25546", 343 :bin{ 411b242943420e4141affe0f43 }, 344 "AB\uc88b%XAF%XFEC", 345 :intvector{ 0, 5, 7, 9, 9, 9, 9, 9, 9, 9, 9, 12 }, 346 :int{1}, :int{1}, "", "&", :bin{""} 347 } 348 349 // truncated, partial escape sequence 350 { 351 "ISO-2022-KR", 352 :bin{ 1b }, "", :intvector{}, 353 :int{1}, :int{1}, "truncated", ".", :bin{ 1b } 354 } 355 { 356 "ISO-2022-KR", 357 :bin{ 1b24 }, "", :intvector{}, 358 :int{1}, :int{1}, "truncated", ".", :bin{ 1b24 } 359 } 360 { 361 "ISO-2022-KR", 362 :bin{ 1b2429 }, "", :intvector{}, 363 :int{1}, :int{1}, "truncated", ".", :bin{ 1b2429 } 364 } 365 // complete escape sequence but nothing else 366 { 367 "ISO-2022-KR", 368 :bin{ 1b242943 }, "", :intvector{}, 369 :int{1}, :int{1}, "", ".", :bin{""} 370 } 371 { 372 "ISO-2022-KR", 373 :bin{ 1b2429430e }, "", :intvector{}, 374 :int{1}, :int{1}, "", ".", :bin{""} 375 } 376 // escape plus ASCII character 377 { 378 "ISO-2022-KR", 379 :bin{ 1b24294341 }, "A", :intvector{ 4 }, 380 :int{1}, :int{1}, "", ".", :bin{""} 381 } 382 // escape plus incomplete DBCS character 383 { 384 "ISO-2022-KR", 385 :bin{ 1b2429430e41 }, "", :intvector{}, 386 :int{1}, :int{1}, "truncated", ".", :bin{ 41 } 387 } 388 // all complete with DBCS character 389 { 390 "ISO-2022-KR", 391 :bin{ 1b2429430e4141 }, "\uc88b", :intvector{ 5 }, 392 :int{1}, :int{1}, "", ".", :bin{""} 393 } 394 // more complicated example 395 { 396 "ISO-2022-KR", 397 :bin{ 411b242943420e4141affe0f43 }, 398 "AB\uc88b%XAF%XFEC", 399 :intvector{ 0, 5, 7, 9, 9, 9, 9, 9, 9, 9, 9, 12 }, 400 :int{1}, :int{1}, "", "&", :bin{""} 401 } 402 // empty segment (using substitution and stop) 403 { 404 "ISO-2022-KR", 405 :bin{ 1b242943610e0f620d0a }, 406 "a\uFFFDb\u000D\u000A", 407 :intvector{ 4, 6, 7, 8, 9 }, 408 :int{1}, :int{1}, "", "?", :bin{""} 409 } 410 { 411 "ISO-2022-KR", 412 :bin{ 1b242943610e0f620d0a }, 413 "a", 414 :intvector{ 4 }, 415 :int{1}, :int{1}, "illesc", ".", :bin{"0f"} 416 } 417 418 // ISO-2022-JP 419 420 // truncated, partial escape sequence 421 { 422 "ISO-2022-JP", 423 :bin{ 1b }, "", :intvector{}, 424 :int{1}, :int{1}, "truncated", ".", :bin{ 1b } 425 } 426 { 427 "ISO-2022-JP-2", 428 :bin{ 1b24 }, "", :intvector{}, 429 :int{1}, :int{1}, "truncated", ".", :bin{ 1b24 } 430 } 431 // complete escape sequence but nothing else 432 { 433 "ISO-2022-JP-2", 434 :bin{ 1b2442 }, "", :intvector{}, 435 :int{1}, :int{1}, "", ".", :bin{""} 436 } 437 // escape plus incomplete DBCS character 438 { 439 "ISO-2022-JP-2", 440 :bin{ 1b244241 }, "", :intvector{}, 441 :int{1}, :int{1}, "truncated", ".", :bin{ 41 } 442 } 443 // all complete with DBCS character 444 { 445 "ISO-2022-JP-2", 446 :bin{ 1b24424141 }, "\u758f", :intvector{ 3 }, 447 :int{1}, :int{1}, "", ".", :bin{""} 448 } 449 // test the G2 designator & SS2 shift 450 { 451 "ISO-2022-JP-2", 452 :bin{ 431b2e46461b244241411b4e4e353f }, "CF\u758f\u039e\u7591", :intvector{ 0, 4, 8, 12, 13 }, 453 :int{1}, :int{1}, "", ".", :bin{""} 454 } 455 // JIS7 with Katakana 456 { 457 "JIS7", 458 :bin{ 41420e41420f4142 }, "AB\uff81\uff82AB", :intvector{ 0, 1, 3, 4, 6, 7 }, 459 :int{1}, :int{1}, "", ".", :bin{""} 460 } 461 // JIS8 with Katakana 462 { 463 "JIS8", 464 :bin{ 41c15c1b284a5cc242 }, "A\uff81\\\xa5\uff82B", :intvector{ 0, 1, 2, 6, 7, 8 }, 465 :int{1}, :int{1}, "", ".", :bin{""} 466 } 467 // empty segment (using substitution and stop) 468 { 469 "ISO-2022-JP", 470 :bin{ 61621b24421b284263640d0a }, 471 "ab\uFFFDcd\u000D\u000A", 472 :intvector{ 0, 1, 5, 8, 9, 10, 11 }, 473 :int{1}, :int{1}, "", "?", :bin{""} 474 } 475 { 476 "ISO-2022-JP", 477 :bin{ 61621b24421b284263640d0a }, 478 "ab", 479 :intvector{ 0, 1 }, 480 :int{1}, :int{1}, "illesc", ".", :bin{"1b2842"} 481 } 482 483 // ISO-2022-CN 484 485 // truncated, partial escape sequence 486 { 487 "ISO_2022,locale=zh,version=1", 488 :bin{ 1b }, "", :intvector{}, 489 :int{1}, :int{1}, "truncated", ".", :bin{ 1b } 490 } 491 { 492 "ISO_2022,locale=zh,version=1", 493 :bin{ 1b24 }, "", :intvector{}, 494 :int{1}, :int{1}, "truncated", ".", :bin{ 1b24 } 495 } 496 { 497 "ISO_2022,locale=zh,version=1", 498 :bin{ 1b2429 }, "", :intvector{}, 499 :int{1}, :int{1}, "truncated", ".", :bin{ 1b2429 } 500 } 501 // complete escape sequence but nothing else 502 { 503 "ISO_2022,locale=zh,version=1", 504 :bin{ 1b242941 }, "", :intvector{}, 505 :int{1}, :int{1}, "", ".", :bin{""} 506 } 507 { 508 "ISO_2022,locale=zh,version=1", 509 :bin{ 1b2429410e }, "", :intvector{}, 510 :int{1}, :int{1}, "", ".", :bin{""} 511 } 512 // escape plus ASCII character 513 { 514 "ISO_2022,locale=zh,version=1", 515 :bin{ 1b24294141 }, "\x41", :intvector{ 4 }, 516 :int{1}, :int{1}, "", ".", :bin{""} 517 } 518 // escape plus incomplete DBCS character 519 { 520 "ISO_2022,locale=zh,version=1", 521 :bin{ 1b2429410e41 }, "", :intvector{}, 522 :int{1}, :int{1}, "truncated", ".", :bin{ 41 } 523 } 524 // all complete with DBCS character 525 { 526 "ISO_2022,locale=zh,version=1", 527 :bin{ 1b2429410e4141 }, "\u4eae", :intvector{ 5 }, 528 :int{1}, :int{1}, "", ".", :bin{""} 529 } 530 /* 531 * ICU 4.4 (ticket #7314) removes mappings for CNS 11643 planes 3..7 532 // ISO-2022-CN-EXT with all subcharsets and shifts and with supplementary code points 533 { 534 "ISO-2022-CN-EXT", 535 :bin{ 1b2429411b242a480e41411b2429457e7c1b4e70341b242b4d1b2429477c341b4f664c2421 }, 536 "\u4eae\u9f82\u56cd\u56cc\U0002a6d6\x30", 537 :intvector{ 9, 15, 19, 29, 33, 33, 35 }, 538 :int{1}, :int{1}, "", ".", :bin{""} 539 } 540 */ 541 542 // illegal and unsupported escape sequences 543 // SS2 without designator: illegal 544 { 545 "ISO-2022-CN-EXT", 546 :bin{ 411b4e2121 }, "\x41", :intvector{ 0 }, 547 :int{1}, :int{1}, "illesc", ".", :bin{ 1b } 548 } 549 // G3 designator: recognized, but not supported for -CN (only for -CN-EXT) 550 { 551 "ISO-2022-CN", 552 :bin{ 411b242b491b4f2121 }, "\x41", :intvector{ 0 }, 553 :int{1}, :int{1}, "unsuppesc", ".", :bin{ 1b242b49 } 554 } 555 // empty segment 1 (using substitution and stop) 556 { 557 "ISO-2022-CN", 558 :bin{ 611b242941620e0f1b242a481b4e6a65630d0a }, 559 "ab\uFFFD\u994Cc\u000D\u000A", 560 :intvector{ 0, 5, 7, 14, 16, 17, 18 }, 561 :int{1}, :int{1}, "", "?", :bin{""} 562 } 563 { 564 "ISO-2022-CN", 565 :bin{ 611b242941620e0f1b242a481b4e6a65630d0a }, 566 "ab", 567 :intvector{ 0, 5 }, 568 :int{1}, :int{1}, "illesc", ".", :bin{"0f"} 569 } 570 // empty segment 2 (using substitution and stop) 571 { 572 "ISO-2022-CN", 573 :bin{ 611b242941620e1b24294768640f630d0a }, 574 "ab\uFFFD\u5F70c\u000D\u000A", 575 :intvector{ 0, 5, 7, 11, 14, 15, 16 }, 576 :int{1}, :int{1}, "", "?", :bin{""} 577 } 578 { 579 "ISO-2022-CN", 580 :bin{ 611b242941620e1b24294768640f630d0a }, 581 "ab", 582 :intvector{ 0, 5 }, 583 :int{1}, :int{1}, "illesc", ".", :bin{"1b242947"} 584 } 585 586 // ISO-2022 SBCS 587 // [U_ENABLE_GENERIC_ISO_2022] 588 // The _generic_ ISO-2022 converter is disabled starting 2003-dec-03 (ICU 2.8). 589 // For details see the icu mailing list from 2003-dec-01 and the ucnv2022.c file. 590 // Language-specific variants of ISO-2022 continue to be available as listed below. 591 //{ 592 // "ISO_2022", 593 // :bin{ 0008090a0d1a1c1f203f415c7d7e7f }, 594 // "\x00\x08\t\n\r\x1a\x1c\x1f ?A\\}~\x7f", 595 // :intvector{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 }, 596 // :int{1}, :int{1}, "", ".", :bin{""} 597 //} 598 599 // HZ-GB-2312 600 601 // empty segment 1 (using substitution and stop) 602 { 603 "HZ-GB-2312", 604 :bin{ 61627e7b7e7d6364 }, 605 "ab\uFFFDcd", 606 :intvector{ 0, 1, 4, 6, 7 }, 607 :int{1}, :int{1}, "", "?", :bin{""} 608 } 609 { 610 "HZ-GB-2312", 611 :bin{ 61627e7b7e7d63640d0a }, 612 "ab", 613 :intvector{ 0, 1 }, 614 :int{1}, :int{1}, "illesc", ".", :bin{"7e7d"} 615 } 616 // empty segment 2 & legal redundant switches (using substitution and stop) 617 { 618 "HZ-GB-2312", 619 :bin{ 61627e7b323b3f557e7b7e7b523b7e7d63647e7d65667e7d7e7d }, 620 "ab\u4E0D\u7A7A\uFFFD\u4E00cdef\uFFFD", 621 :intvector{ 0, 1, 4, 6, 10, 12, 16, 17, 20, 21, 24 }, 622 :int{1}, :int{1}, "", "?", :bin{""} 623 } 624 { 625 "HZ-GB-2312", 626 :bin{ 61627e7b323b3f557e7b7e7b523b7e7d63647e7d65667e7d7e7d }, 627 "ab\u4E0D\u7A7A", 628 :intvector{ 0, 1, 4, 6 }, 629 :int{1}, :int{1}, "illesc", ".", :bin{"7e7b"} 630 } 631 632 // DBCS-only extensions 633 { 634 "ibm-970", 635 :bin{ 617eece9b2eb }, 636 "\x61\x7e\u4e00\ub000", 637 :intvector{ 0, 1, 2, 4 }, 638 :int{1}, :int{1}, "", "?", :bin{""} 639 } 640 641 { 642 "ibm-971", 643 :bin{ 617eece9b2eb }, 644 "\ufffd\u4e00\ub000", 645 :intvector{ 0, 2, 4 }, 646 :int{1}, :int{1}, "", "?", :bin{""} 647 } 648 649 { 650 "ibm-16684", 651 :bin{ 430e4395ecc1404042e1 }, 652 "\ufffd\u30C8\u30C8\u309A\u3000\u20ac", 653 :intvector{ 0, 2, 4, 4, 6, 8 }, 654 :int{1}, :int{0}, "", "?", :bin{""} 655 } 656 657 { 658 "ibm-1399", 659 :bin{ 430e4395ecc140400fe1 }, 660 "\uff62\u30C8\u30C8\u309A\u3000\u20ac", 661 :intvector{ 0, 2, 4, 4, 6, 9 }, 662 :int{1}, :int{0}, "", "?", :bin{""} 663 } 664 665 // extensions 666 { 667 "ibm-1390", 668 :bin{ 430e4395ecc1 }, 669 "\uff63\u30C8\u30C8\u309A", 670 :intvector{ 0, 2, 4, 4 }, 671 :int{1}, :int{0}, "", "?", :bin{""} 672 } 673 674 { 675 "ibm-16684", 676 :bin{ ececec8bec8cec8d4386ecb5ecb6ecb7 }, 677 "\ufffd\u31f6\u31f7\u31f8\u30ab\u304b\u309a\u304d\u309a\u304f\u309a", 678 :intvector{ 0, 2, 4, 6, 8, 10, 10, 12, 12, 14, 14 }, 679 :int{1}, :int{0}, "", "?", :bin{""} 680 } 681 682 { 683 "ibm-1390", 684 :bin{ 43860eececec8bec8cec8d4386ecb5ecb6ecb7ecc10fec }, 685 "\uff63\uff76\ufffd\u31f6\u31f7\u31f8\u30ab\u304b\u309a\u304d\u309a\u304f\u309a\u30C8\u309A\x1a", 686 :intvector{ 0, 1, 3, 5, 7, 9, 11, 13, 13, 15, 15, 17, 17, 19, 19, 22 }, 687 :int{1}, :int{0}, "", "?", :bin{""} 688 } 689 690 { 691 "*test3", 692 :bin{ 00050601020b0701020a01020c }, 693 "\u20ac\x05\x06\x0b\U00101234\U00023456\ufffd", 694 :intvector{ 0, 1, 2, 3, 6, 6, 7, 7, 10 }, 695 :int{1}, :int{0}, "", "?", :bin{""} 696 } 697 698 // test mapping to sequence of multiple Unicode characters which includes nonBMP (ticket #9235) 699 { 700 "*test3", 701 :bin{ 05070001020e050501020c06 }, 702 "\x05\U00101234\U00050005\u00c4\u00c4\U00101234\x05\x06", 703 :intvector{ 0, 1, 1, 1, 1, 7, 7, 7, 7, 7, 11 }, 704 :int{1}, :int{0}, "", "?", :bin{""} 705 } 706 707 // normal conversions 708 { 709 "UTF-16LE", 710 :bin{ 310000d801dc00d902dc320000d8330001dc3400 }, 711 "1\U00010001\U000500022\ufffd3\ufffd4", 712 :intvector{ 0, 2, 2, 6, 6, 10, 12, 14, 16, 18 }, 713 :int{1}, :int{0}, "", "?", :bin{""} 714 } 715 { "UTF-16LE", :bin{ 00 }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ 00 } } 716 { "UTF-16LE", :bin{ 00d800 }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ 00d800 } } 717 718 { 719 "UTF-16BE", 720 :bin{ 0031d800dc01d900dc020032d8000033dc010034 }, 721 "1\U00010001\U000500022\ufffd3\ufffd4", 722 :intvector{ 0, 2, 2, 6, 6, 10, 12, 14, 16, 18 }, 723 :int{1}, :int{0}, "", "?", :bin{""} 724 } 725 { "UTF-16BE", :bin{ 00 }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ 00 } } 726 { "UTF-16BE", :bin{ d800dc }, "", :intvector{}, :int{1}, :int{0}, "truncated", ".", :bin{ d800dc } } 727 728 // e4b8 is a partial sequence 729 { "UTF-8", :bin{ 31e4ba8ce4b8 }, "1\u4e8c", :intvector{ 0, 1 }, :int{1}, :int{0}, "truncated", ".", :bin{ e4b8 } } 730 { "UTF-8", :bin{ 31e4ba8ce4b8 }, "1\u4e8c\ufffd", :intvector{ 0, 1, 4 }, :int{1}, :int{0}, "", "?", :bin{""} } 731 732 // LMBCS with escape callback (1292a0 is unassigned) 733 { 734 "LMBCS", 735 :bin{ 12c9501292a01292a1 }, 736 "\u4e2e%X12%X92%XA0\ue5c4", 737 :intvector{ 0, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 6 }, 738 :int{1}, :int{0}, "", "&", :bin{""} 739 } 740 { 741 "LMBCS", 742 :bin{ 61012981a00f270f91140a7414f60214d84d14dc561088a0 }, 743 "\u0061\u2013\u00fc\u00e1\u0007\u0091\u0a74\u0200\ud84d\udc56\u5516", 744 :intvector{ 0, 1, 3, 4, 5, 7, 9, 12, 15, 18, 21 }, 745 :int{1}, :int{0}, "", ".", :bin{""} 746 } 747 748 // IMAP-mailbox-name with SUB 749 // a<DEL> a&AB~ a&AB\x0c a&AB- a&AB. a&. 750 { 751 "IMAP-mailbox-name", 752 :bin{ 617f612641427e612641420c612641422d612641422e61262e }, 753 "a\ufffda\ufffda\ufffda\ufffda\ufffda\ufffd", 754 :intvector{ 0, 1, 2, 4, 7, 9, 12, 14, 17, 19, 22, 23 }, 755 :int{1}, :int{0}, "", "?", :bin{""} 756 } 757 758 // using testdata_test1.cnv 759 { "*test1", :bin{ 000506070809 }, "\u20ac\x05\x06\U00101234\ufffd\ufffd", :intvector{ 0, 1, 2, 3, 3, 4, 5 }, :int{1}, :int{0}, "", "", :bin{""} } 760 761 // surrogates in CESU-8 762 { "CESU-8", :bin{ eda080eda081edb081 }, "\ud800\U00010401", :intvector{ 0, 3, 6 }, :int{1}, :int{0}, "", "", :bin{""} } 763 // e080 is a partial sequence 764 { "UTF-8", :bin{ 31ffe4ba8ce08061 }, "1\ufffd\u4e8c\ufffda", :intvector{ 0, 1, 2, 5, 7 }, :int{0}, :int{0}, "", "", :bin{ e080 } } 765 // fbbfbfbfbf exceedes U+10ffff 766 { "UTF-8", :bin{ 31fbbfbfbfbf61 }, "1\ufffda", :intvector{ 0, 1, 6 }, :int{0}, :int{0}, "", "", :bin{ fbbfbfbfbf } } 767 768 // lead byte a2 without trail byte 769 { "ibm-1363", :bin{ a2aea2 }, "\u00a1", :intvector{ 0 }, :int{1}, :int{0}, "truncated", ".", :bin{ a2 } } 770 { "ibm-1363", :bin{ a2aea2 }, "\u00a1\u001a", :intvector{ 0, 2 }, :int{1}, :int{0}, "", "?", :bin{""} } 771 772 // simple sample, no error handling 773 { "UTF-8", :bin{ 61F48FBFBF }, "a\U0010FFFF", :intvector{ 0, 1, 1 }, :int{1}, :int{0}, "", "", :bin{""} } 774 // Headers { "charset", "bytes", "unicode", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidChars" } 775 { 776 "iscii-dev", 777 :bin{ EF4BC0E9BFE9E8D80AEF4AC0D4BFD4E8D80AEF4838B30AEF4939B30AEF4A3AB30AEF4B3BB30A}, 778 "\u0A5C\u0A4D\u0A39\u0A5C\u0A4D\u0A39\u000A" /* Gurmukhi test */ 779 "\u0AA2\u0AB5\u0AA1\u0AB5\u0ACD\u0AB9\u000A" /* Gujarati test */ 780 "\u0038\u0C95\u000A" /* Kannada test */ 781 "\u0039\u0D15\u000A" /* Malayalam test */ 782 "\u003A\u0A95\u000A" /* Gujarati test */ 783 "\u003B\u0A15\u000A" /* Punjabi test */, 784 :intvector { 2, 2, 2, 4, 6, 7, 8, 11, 12, 13, 14, 15, 16, 17, 20, 21, 22, 25, 26, 27, 30, 31, 32, 35, 36, 37 }, 785 :int{1}, :int{0}, "", "", :bin{ "" } 786 } 787 { 788 "iscii-gur", 789 :bin{3BB30AC0E9BFE9E8D80AEF43C0E9BFE9E8D80A3BB30AEF403BB30A}, 790 "\u003b\u0a15\u000a" /* Easy characters */ 791 "\u0a5c\u0a4d\u0a39\u0a5c\u0a4d\u0a39\u000a" /* Gurmukhi test */ 792 "\u09dd\u09dc\u09cd\u09b9\u000a" /* Switch script: to Bengali*/ 793 "\u003b\u0a15\u000a" /* Easy characters - new line" so should default!*/ 794 "\u003b\u0a15\u000a", /* Back to Gurmukhi*/ 795 :intvector { 0, 1, 2, 3, 3, 3, 5, 7, 8, 9, 13, 15, 16, 17, 18, 19, 20, 21, 24, 25, 26 }, 796 :int{1}, :int{0}, "", "", :bin{ "" } 797 } 798 // Test Gurmukhi (Bindi Tippi and Consonant Cluster) 799 { 800 "iscii-gur", 801 :bin{EF4BB3A2A2B3E8B3C0E9BFE9E8D8}, 802 "\u0a15\u0a70\u0a02\u0a71\u0a15\u0a5c\u0a4d\u0a39\u0a5c\u0a4d\u0a39", 803 :intvector { 2, 3, 4, 5, 5, 8, 8, 8, 10, 12, 13 }, 804 :int{1}, :int{0}, "", ".", :bin{ "" } 805 } 806 { // Verify Supplementary support 807 "Big5-HKSCS", 808 :bin{ fcfcfcfd }, 809 "\U000233E6\U00026DA0", 810 :intvector{ 0, 0, 2, 2 }, 811 :int{1}, :int{0}, "", "?", :bin{""} 812 } 813 { // Verify non-supplementary support 814 "big5-hkscs:unicode3.0", 815 :bin{ fcfcfcfd }, 816 "\uE1D4\uE1D5", 817 :intvector{ 0, 2 }, 818 :int{1}, :int{0}, "", "?", :bin{""} 819 } 820 { // Verify gb18030 enumeration 821 "gb18030", 822 :bin{ 8130D2398130D3308136A5318136A532 }, 823 "\u0450\u0452\u200F\u2011", 824 :intvector{ 0, 4, 8, 12 }, 825 :int{1}, :int{0}, "", "?", :bin{""} 826 } 827 { 828 "x11-compound-text", 829 :bin{ 1b242944b5ac1b2d41a5e31b2d43d5f51b2d4dd01b2d41411b2d43bc1b2d42ff1b2d54df1b2d44c0b31b2d46b41b2d47b01b2d48e01b2d4ca1 }, 830 "\u54A1\u00A5\u00E3\u0120\u0121\u011E\u0041\u0135\u02D9\u0E3F\u0100\u0157\u0384\u0660\u05D0\u0401", 831 :intvector{ }, 832 :int{1}, :int{0}, "", "?", :bin{""} 833 } 834 // Improve coverage of ISCII 835 { 836 "iscii-bng", 837 :bin{ f0a0f0b0b8 }, 838 "", 839 :intvector{}, 840 :int{1}, :int{0}, "", "0", :bin{""} 841 } 842 // Test iso-2022-jp-2 miscellaneous symbols 843 { 844 "iso-2022-jp-2", 845 :bin{ 1b242843224f224e1b2842 }, 846 "\u260E\u260F", 847 :intvector{ 4, 6 }, 848 :int{1}, :int{0}, "", ".", :bin{""} 849 } 850 851 // Improve Code Coverage for BOCU-1 852 { 853 "BOCU-1", 854 :bin{ 91fbc555fd6349 }, 855 "\u0041\ud841\ud888\udc81", 856 :intvector{ 0, 1, 4, 4 }, 857 :int{1}, :int{0}, "", ".", :bin{""} 858 } 859 { 860 "BOCU-1", 861 :bin{ fbeda44ff0fe189bb821f05926 }, 862 "\ufe88\ufe70\udbff\udfff\u0061", 863 :intvector{}, 864 :int{1}, :int{0}, "", ".", :bin{""} 865 } 866 { 867 "BOCU-1", 868 :bin{ 5b4bccf9 }, 869 "\u000b", 870 :intvector{}, 871 :int{1}, :int{0}, "", "0", :bin{""} 872 } 873 { 874 "BOCU-1", 875 :bin{ fe0053c6 }, 876 "\u0003\u0076", 877 :intvector{}, 878 :int{1}, :int{0}, "", "0", :bin{""} 879 } 880 881 //Improve code coverage for SCSU 882 { 883 "SCSU", 884 :bin{ 0fd899dc7fd888dc99e041424361 }, 885 "\ud899\udc7f\ud888\udc99\u0041\u0042\u0043\u0061", 886 :intvector{}, 887 :int{1}, :int{0}, "", "0", :bin{""} 888 } 889 { 890 "SCSU", 891 :bin{ 41df1281035f10df1b03df1c88800bbfffff }, 892 "\u0041\u00df\u0401\u015f\u00df\u01df\uf000\udbff\udfff", 893 :intvector{}, 894 :int{1}, :int{0}, "", ".", :bin{""} 895 } 896 { 897 "SCSU", 898 :bin{ 1b9a1b541bb2411bfd1b0041 }, 899 "", 900 :intvector{}, 901 :int{1}, :int{0}, "", ".", :bin{""} 902 } 903 { 904 "SCSU", 905 :bin{ 0f6441b413a733f2 }, 906 "\u6441\ub413\ua733", 907 :intvector{}, 908 :int{1}, :int{0}, "illegal", ".", :bin{ f2 } 909 } 910 911 //Improve code coverage for MBCS 912 { 913 "*test5", 914 :bin{ 0506 }, 915 "\ufffd\x06", 916 :intvector{}, 917 :int{1}, :int{0}, "", "?", :bin{""} 918 } 919 { 920 "ibm-1390,swaplfnl", 921 :bin{ 430e4395ecc140400fc1e115 }, 922 "\uff63\u30C8\u30C8\u309A\u3000\x41\u20ac\x0a", 923 :intvector{ 0, 2, 4, 4, 6, 9, 10, 11 }, 924 :int{1}, :int{0}, "", "?", :bin{""} 925 } 926 /* 927 * ICU 4.4 (ticket #7314) removes mappings for CNS 11643 planes 3..7 928 { 929 "ISO-2022-CN-EXT", 930 :bin{ 1b242b4d1b4f66791b242b4d1b4f216a }, 931 "\u3667", 932 :intvector{ 14 }, 933 :int{1}, :int{0}, "", "0", :bin{""} 934 } 935 */ 936 { 937 "*test1bmp", 938 :bin{ 060a05 }, 939 "\u0066\ufffd\u0065", 940 :intvector{ 0, 2, 2 }, 941 :int{0}, :int{1}, "", "?", :bin{""} 942 } 943 { 944 "*test1bmp", 945 :bin{ 060708 }, 946 "\u0066", 947 :intvector{}, 948 :int{1}, :int{0}, "invalid", ".", :bin{ 07 } 949 } 950 { 951 "*test5", 952 :bin{ 010304 }, 953 "\u0034\ufffd", 954 :intvector{}, 955 :int{1}, :int{0}, "", "?", :bin{""} 956 } 957 { 958 "*test1", 959 :bin{ 0a0b }, 960 "", 961 :intvector{}, 962 :int{1}, :int{0}, "", "0", :bin{""} 963 } 964 { 965 "*test1bmp", 966 :bin{ 0c06 }, 967 "\u0066", 968 :intvector{}, 969 :int{0}, :int{0}, "", ".", :bin{""} 970 } 971 { 972 "*test5", 973 :bin{ 0906 }, 974 "\udbc8\udf45\u0006", 975 :intvector{}, 976 :int{0}, :int{0}, "", ".", :bin{""} 977 } 978 { 979 "ibm-16684", 980 :bin{ 0e }, 981 "", 982 :intvector{}, 983 :int{0}, :int{0}, "illegal", ".", :bin{ 0e } 984 } 985 { 986 "UTF-7", 987 :bin{ 2b414b4d2d492b414b4d4170412d }, 988 "\u00a3I\u00a3\u00a4", 989 :intvector{ 1,5,7,9 }, 990 :int{0}, :int{0}, "", ".", :bin{""} 991 } 992 { 993 "x11-compound-text", 994 :bin{ 1e6ddc9b26bc10801bbcad50a040fc }, 995 "\u001e\u006d\u00dc\u009b\u0026\u00bc\u0010\u0080\ufffd\u00bc\u00ad\u0050\u00a0\u0040\u00fc", 996 :intvector{ }, 997 :int{1}, :int{0}, "", "?", :bin{""} 998 } 999 } 1000 } 1001 1002 // --------------------------------------------------------------------- *** 1003 1004 fromUnicode { 1005 Headers { "charset", "unicode", "bytes", "offsets", "flush", "fallbacks", "errorCode", "callback", "invalidUChars" } 1006 Cases { 1007 // Test ticket 9602: Add "good one-way" mapping type (|4). 1008 // Such mappings are used regardless of the fallback flag. 1009 { 1010 "+*test3", "##\uFE0E#\uFE0F", 1011 :bin{ 010204010204010204 }, :intvector{ 0,0,0,1,1,1,3,3,3 }, 1012 :int{1}, :int{0}, // no fallbacks 1013 "", "?", "" 1014 } 1015 { 1016 "+*test3", "##\uFE0E#\uFE0F", 1017 :bin{ 010204010204010204 }, :intvector{ 0,0,0,1,1,1,3,3,3 }, 1018 :int{1}, :int{1}, // with fallbacks 1019 "", "?", "" 1020 } 1021 // Test ticket 6789: implement Java-compatible Unicode, UnicodeBig and UnicodeLittle converters 1022 // For details about these encodings see convrtrs.txt. 1023 // Standard UTF-16BE 1024 { "UTF-16BE", "a", :bin{ 0061 }, :intvector{ 0,0 }, :int{1}, :int{0}, "", "?", "" } 1025 // Java "UnicodeBig" writes a BOM 1026 { "UnicodeBig", "a", :bin{ feff0061 }, :intvector{ -1,-1,0,0 }, :int{1}, :int{0}, "", "?", "" } 1027 // Standard UTF-16LE 1028 { "UTF-16LE", "a", :bin{ 6100 }, :intvector{ 0,0 }, :int{1}, :int{0}, "", "?", "" } 1029 // Java "UnicodeLittle" writes a BOM 1030 { "UnicodeLittle", "a", :bin{ fffe6100 }, :intvector{ -1,-1,0,0 }, :int{1}, :int{0}, "", "?", "" } 1031 1032 // Test ticket 7704: implement Java-compatible "UTF-16" converter. 1033 // Same as standard UTF-16 but fromUnicode always writes big-endian byte stream. 1034 { "+UTF-16,version=2", "a", :bin{ feff0061 }, :intvector{ -1,-1,0,0 }, :int{1}, :int{0}, "", "?", "" } 1035 1036 // Test bug 6071 (1:2 Unicode:charset SBCS mapping). 1037 { 1038 "*test1bmp", 1039 "e@t", 1040 :bin{ 05000709 }, 1041 :intvector{ 0,1,2,2 }, 1042 :int{1}, :int{0}, "", "?", "" 1043 } 1044 // improve coverage of ISO-2022-JP converter with hardcoded JIS X 0201 and 1045 // using the Shift-JIS table for JIS X 0208 (ticket #5797) 1046 { 1047 "ISO-2022-JP", 1048 "\u203e\xa5\u4e00\ufa10\u6f3e\u0391", 1049 // Google Specific Patch for alternatate 2022 conversion table. 1050 // :bin{ 1b284a7e5c1b2442306c222e5f2126211b2842 }, 1051 :bin{ 1b284a7e5c1b2442306c79605f2126211b2842 }, 1052 // End Google specific patch. 1053 :intvector{ 0,0,0,0,1,2,2,2,2,2,3,3,4,4,5,5,5,5,5 }, 1054 :int{1}, :int{0}, "", "?=\u3013", "" // U+3013 Geta Mark converts to 222e 1055 } 1056 // Verify that mappings that would result in byte values outside 20..7F (for SBCS) 1057 // or 21..7E (for DBCS) are not used. 1058 // ibm-9005_X110-2007.ucm (ISO 8859-7, <ESC>.F=1b2e46): 1059 // <U009F> \x9F |0 (also in ISO 8859-1) 1060 // <U0387> \xB7 |1 1061 // windows-949-2000 (KSC_5601, <ESC>$(C=1b242843): 1062 // <UC829> \xA0\xA1 |0 1063 // <UD4FE> \xC0\x41 |0 1064 // <UD79D> \xC8\xFE |0 1065 { 1066 "JIS8", // =ISO_2022,locale=ja,version=4 1067 "\u009f\u0387\uc829\ud4fe\ud79d", 1068 :bin{ 1a1b2e461b4e371a1a1b242843487e1b2842 }, 1069 :intvector{ 0,1,1,1,1,1,1,2,3,4,4,4,4,4,4,4,4,4 }, 1070 :int{1}, :int{1}, "", "?", "" 1071 } 1072 // Ticket 5483: ISO 2022 converter incorrectly using fallback mapping 1073 // Verify that a roundtrip mapping is used even when a fallback mapping is 1074 // available in the current state. 1075 // U+FF61 is handled in code 1076 // jisx-208.ucm (<ESC>$B=1b2442): 1077 // <U30FE> \x21\x34 |0 1078 // <UFF5D> \x21\x51 |0 and 1079 // ibm-897_P100-1995.ucm (JIS X 0201, <ESC>(J=1b284a): 1080 // <UFF5D> \x7D |1 1081 // ibm-9005_X110-2007.ucm (ISO 8859-7, <ESC>.F=1b2e46): 1082 // <U03D5> \xF6 |1 1083 // <U2015> \xAF |0 1084 // <UFF5D> \x7D |1 (not legal for ISO 2022) 1085 // windows-949-2000 (KSC_5601, <ESC>$(C=1b242843): 1086 // <UAC00> \xB0\xA1 |0 1087 // <UFF5D> \xA3\xFD |0 1088 // <U223C> \xA1\xAD |0 (in extension table) 1089 { 1090 "JIS8", // =ISO_2022,locale=ja,version=4 1091 "a\uff61\u03d5\uff5d\uac00\u223c\uff5d\u30fe\uff5d", // Make it switch to ISO-8859-7, KSC 5601 and JIS X 0208. 1092 :bin{ 61a11b2e461b4e761b244221511b2428433021212d237d1b2442213421511b2842 }, 1093 :intvector{ 0,1,2,2,2,2,2,2,3,3,3,3,3,4,4,4,4,4,4,5,5,6,6,7,7,7,7,7,8,8,8,8,8 }, 1094 :int{1}, :int{1}, "", "?", "" 1095 } 1096 // Code coverage for UTF-8->SBCS conversion (ucnv_convertEx()). 1097 // Test code path for non-roundtripping ASCII characters 1098 // (try EBCDIC SBCS, and IBM PC SBCS with control code rotation). 1099 { 1100 "ibm-37", 1101 "a\x85c", 1102 :bin{ 811583 }, 1103 :intvector{ 0,1,2 }, 1104 :int{1}, :int{0}, "", "?", "" 1105 } 1106 { 1107 "ibm-850", 1108 "a\x1ac", 1109 :bin{ 617f63 }, 1110 :intvector{ 0,1,2 }, 1111 :int{1}, :int{0}, "", "?", "" 1112 } 1113 // Code coverage for UTF-8->DBCS conversion (ucnv_convertEx()). 1114 // Test code path for non-roundtripping ASCII characters 1115 // (try IBM PC DBCS with control code rotation). 1116 { 1117 "ibm-943", 1118 "a\x1ac\u30a1\x7ff", 1119 :bin{ 617f6383401c66 }, 1120 :intvector{ 0,1,2,3,3,4,5 }, 1121 :int{1}, :int{0}, "", "?", "" 1122 } 1123 // SCSU regression test. 1124 { 1125 "SCSU", 1126 "1\U00010001\u0085\U000500022\ud8003\udc014\ue001", 1127 :bin{ 310be0008102050fd900dc02e7320efffd330efffd34186881 }, 1128 :intvector{ 0,1,1,1,1,3,3,4,4,4,4,4,6,6,7,7,7,8,9,9,9,10,11,11,11 }, 1129 :int{1}, :int{0}, "", "?", "" 1130 } 1131 // LMBCS 1132 { 1133 "lmbcs", 1134 "\u0061\u2013\u00fc\u00e1\u0007\u0091\u0a74\u0200\ud84d\udc56\u5516", 1135 :bin { 61039681a00f270f91140a7414f60214d84d14dc561088a0 }, 1136 :intvector{ 0, 1, 1, 2, 3, 4, 4, 5, 5, 6, 6, 6, 7, 7, 7, 8, 8, 8, 9, 9, 9, 10, 10, 10 }, 1137 :int{1}, :int{0}, "", ".", "" 1138 } 1139 // Test substitution strings. 1140 { 1141 "windows-1252", // stateless MBCS with WriteSub(), should internally set char * 1142 "a\ufdd0c", 1143 :bin{ 61402421402463 }, 1144 :intvector{ 0,1,1,1,1,1,2 }, 1145 :int{1}, :int{0}, "", "?=@$!@$", "" 1146 } 1147 { 1148 "windows-1252", 1149 "1\U00010001\u0085\U000500022\ud8003\udc014\ue001", 1150 :bin{ 311a1a1a321a331a341a }, 1151 :intvector{ 0,1,3,4,6,7,8,9,10,11 }, 1152 :int{1}, :int{0}, "", "?", "" 1153 } 1154 { 1155 "windows-1252", 1156 "\uD87E", // lone surrogate can cause an offset overflow 1157 :bin{ 1a }, 1158 :intvector{ 0 }, 1159 :int{1}, :int{0}, "", "?", "" 1160 } 1161 { 1162 "windows-1252", 1163 "\uD87E", // lone surrogate can cause an offset overflow 1164 :bin{ 6875683f }, 1165 :intvector{ 0,0,0,0 }, 1166 :int{1}, :int{0}, "", "?=huh?", "" // Use a long substitution character 1167 } 1168 { 1169 "*test4", 1170 "\u30ab", // An incomplete multi-codepoint character 1171 :bin{ ff }, 1172 :intvector{ 0 }, 1173 :int{1}, :int{0}, "", "?", "" 1174 } 1175 { 1176 "ibm-930", // stateful MBCS 1177 "a\ufdd0\u4e00\ufdd0e", 1178 :bin{ 620e4bce0f400e45414bce0f4066 }, 1179 :intvector{ 0,1,1,1,1,1,2,2,2,3,3,3,3,4 }, 1180 :int{1}, :int{0}, "", "?=\u4e01 ", "" 1181 } 1182 { 1183 "iso-2022-jp", 1184 "a\x1bc", // Unicode ESC must not occur as a character 1185 :bin{ 6163 }, 1186 :intvector{ 0,2 }, 1187 :int{1}, :int{0}, "", "?=", "" // empty substitution string 1188 } 1189 { 1190 "iso-2022-cn", 1191 "a\x1bc", // Unicode ESC must not occur as a character 1192 :bin{ 61202063 }, 1193 :intvector{ 0,1,1,2 }, 1194 :int{1}, :int{0}, "", "?= ", "" 1195 } 1196 { 1197 "iso-2022-cn", 1198 "a\x1bc", // Unicode ESC must not occur as a character 1199 :bin{ 611b2429410e523b0f2063 }, 1200 :intvector{ 0,1,1,1,1,1,1,1,1,1,2 }, 1201 :int{1}, :int{0}, "", "?=\u4e00 ", "" 1202 } 1203 { 1204 "us-ascii", 1205 "a\x85c", 1206 :bin{ 61402421402463 }, 1207 :intvector{ 0,1,1,1,1,1,2 }, 1208 :int{1}, :int{0}, "", "?=@$!@$", "" 1209 } 1210 // ISO 2022-CN: test a single-byte subchar, j5171 1211 { 1212 "iso-2022-cn", 1213 "a\x1bc", // Unicode ESC must not occur as a character 1214 :bin{ 612163 }, 1215 :intvector{ 0,1,2 }, 1216 :int{1}, :int{0}, "", "?\x00\x21", "" 1217 } 1218 // UTF-16/32: do not output a BOM if there is no data at all 1219 { 1220 "UTF-16", 1221 "", 1222 :bin{ "" }, 1223 :intvector{ }, 1224 :int{1}, :int{1}, "", "?", "" 1225 } 1226 { 1227 "UTF-32", 1228 "", 1229 :bin{ "" }, 1230 :intvector{ }, 1231 :int{1}, :int{1}, "", "?", "" 1232 } 1233 1234 // do not convert SO/SI/ESC 1235 { 1236 "iso-2022-jp", 1237 "A\x0eB\x0f\x09\x1bC", 1238 :bin{ 411a421a091a43 }, 1239 :intvector{ 0,1,2,3,4,5,6 }, 1240 :int{1}, :int{1}, "", "?", "" 1241 } 1242 { 1243 "iso-2022-cn", 1244 "A\x0eB\x0f\x09\x1bC", 1245 :bin{ 411a421a091a43 }, 1246 :intvector{ 0,1,2,3,4,5,6 }, 1247 :int{1}, :int{1}, "", "?", "" 1248 } 1249 { 1250 "iso-2022-kr", 1251 "A\x0eB\x0f\x09\x1bC", 1252 :bin{ 1b242943411a421a091a43 }, 1253 :intvector{ -1,-1,-1,-1,0,1,2,3,4,5,6 }, 1254 :int{1}, :int{1}, "", "?", "" 1255 } 1256 { 1257 "ibm-25546", 1258 "A\x0eB\x0f\x09\x1bC", 1259 :bin{ 1b242943411a421a091a43 }, 1260 :intvector{ -1,-1,-1,-1,0,1,2,3,4,5,6 }, 1261 :int{1}, :int{1}, "", "?", "" 1262 } 1263 1264 // test ISO 8859-1/7 vs. JIS X 0201 1265 { 1266 "ISO-2022-JP-2", 1267 "?@A\u00e4\u03b1\u203EB", 1268 :bin{ 3f40411B2E411B4E641B2E461B4E611B284A7E421B2842 }, 1269 :intvector{ 0,1,2,3,3,3,3,3,3,4,4,4,4,4,4,5,5,5,5,6,6,6,6 }, 1270 :int{1}, :int{1}, "", "?", "" 1271 } 1272 1273 // Improve ucnv_ext.c code coverage: 1274 // There will be a partial match up to the lead surrogate of U+603ff 1275 // which then results in one more unit in the prefetch buffer 1276 // than the match length when converting one code unit at a time. 1277 // See ucnv_extContinueMatchFromU() comment 1278 // "the match did not use all of preFromU[] - keep the rest for replay" 1279 { 1280 "*test3", 1281 "\U00101234\U00101234\U00050005\U000603ff", 1282 :bin{ 07070001020e05ff }, 1283 :intvector{ 0, 2, 2, 2, 2, 2, 2, 6 }, 1284 :int{1}, :int{0}, "", "?", "" 1285 } 1286 1287 // test that ISO-2022-JP encodes ASCII as itself 1288 { 1289 "ISO-2022-JP", 1290 "?@ABCDEFG", 1291 :bin{ 3f4041424344454647 }, 1292 :intvector{ 0,1,2,3,4,5,6,7,8 }, 1293 :int{1}, :int{1}, "", "?", "" 1294 } 1295 // test that ISO-2022-CN encodes ASCII as itself 1296 { 1297 "ISO-2022-CN", 1298 "?@ABCDEFG", 1299 :bin{ 3f4041424344454647 }, 1300 :intvector{ 0,1,2,3,4,5,6,7,8 }, 1301 :int{1}, :int{1}, "", "?", "" 1302 } 1303 1304 // moved from cintltst /tsconv/nccbtst/TestSkipCallBack 1305 { 1306 "iso-2022-jp", 1307 "\u3000\xe9\u3001", 1308 :bin{ 1b2442212121221b2842 }, 1309 :intvector{ 0,0,0,0,0,2,2,2,2,2 }, 1310 :int{1}, :int{1}, "", "0", "" 1311 } 1312 // moved from cintltst /tsconv/nccbtst/TestSubCallBack 1313 { 1314 "iso-2022-jp", 1315 "A\xe9B\xe9\u3000", 1316 :bin{ 411a421a1b244221211b2842 }, 1317 :intvector{ 0,1,2,3,4,4,4,4,4,4,4,4 }, 1318 :int{1}, :int{1}, "", "?", "" 1319 } 1320 // moved from cintltst /tsconv/nccbtst/TestSubWithValueCallBack 1321 { 1322 "iso-2022-jp", 1323 "A\xe9B\xe9\u3000", 1324 :bin{ 41255530304539422555303045391b244221211b2842 }, 1325 :intvector{ 0,1,1,1,1,1,1,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4 }, 1326 :int{1}, :int{1}, "", "&", "" 1327 } 1328 { 1329 "iso-2022-cn", 1330 "\u4e00\u3712\u4e01", 1331 :bin{ 1b2429410e523b0f2555333731320e36210f }, 1332 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,1,2,2,2,2 }, 1333 :int{1}, :int{1}, "", "&", "" 1334 } 1335 { 1336 "iso-2022-cn", 1337 "A\u3712\u4e00", 1338 :bin{ 412555333731321b2429410e523b0f }, 1339 :intvector{ 0,1,1,1,1,1,1,2,2,2,2,2,2,2,2 }, 1340 :int{1}, :int{1}, "", "&", "" 1341 } 1342 { 1343 "iso-2022-cn", 1344 "\u3000\u3712\u3001", 1345 :bin{ 1b2429410e21210f2555333731320e21220f }, 1346 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,1,2,2,2,2 }, 1347 :int{1}, :int{1}, "", "&", "" 1348 } 1349 1350 // moved from cintltst /tsconv/nucnvtst/TestJIS 1351 { 1352 "JIS", 1353 "\uFF81\uFF82\u30EC\u30ED\u30EE\u30EF\uFF93\uFF94\uFF95\uFF96\uFF97\uFF98", 1354 :bin{ 1b244225412544256c256d256e256F25622564256625682569256a1b2842 }, 1355 :intvector{ 0,0,0,0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,8,8,9,9,10,10,11,11,11,11,11 }, 1356 :int{1}, :int{1}, "", "?", "" 1357 } 1358 { 1359 "JIS7", 1360 "\uFF81\uFF82\u30EC\u30ED\u30EE\u30EF\uFF93\uFF94\uFF95\uFF96\uFF97\uFF98", 1361 :bin{ 0e41420f1b2442256c256d256e256F0e5354555657580f1b2842 }, 1362 :intvector{ 0,0,1,2,2,2,2,2,2,3,3,4,4,5,5,6,6,7,8,9,10,11,11,11,11,11 }, 1363 :int{1}, :int{1}, "", "?", "" 1364 } 1365 { 1366 "JIS8", 1367 "\uFF81\uFF82\u30EC\u30ED\u30EE\u30EF\uFF93\uFF94\uFF95\uFF96\uFF97\uFF98", 1368 :bin{ C1C21b2442256c256d256e256F1b284AD3D4D5D6D7D81b2842 }, 1369 :intvector{ 0,1,2,2,2,2,2,3,3,4,4,5,5,6,6,6,6,7,8,9,10,11,11,11,11 }, 1370 :int{1}, :int{1}, "", "?", "" 1371 } 1372 1373 // moved from cintltst /tsconv/ncnvtst/TestErrorBehaviour 1374 { 1375 "iso-2022-jp", 1376 "\u3000\x50\udc01\u3001", 1377 :bin{ 1B244221211B2842501A1B24422122 }, 1378 :intvector{ 0,0,0,0,0,1,1,1,1,2,3,3,3,3,3 }, 1379 :int{0}, :int{1}, "", "?", "\udc01" 1380 } 1381 { 1382 "iso-2022-jp", 1383 "\u3000\x50\udc01\u3001", 1384 :bin{ 1B244221211B2842501A1B244221221b2842 }, 1385 :intvector{ 0,0,0,0,0,1,1,1,1,2,3,3,3,3,3,3,3,3 }, 1386 :int{1}, :int{1}, "", "?", "" 1387 } 1388 { 1389 "iso-2022-kr", 1390 "\x61\u4e00\udc01\u4e00", 1391 :bin{ 1b242943610e6c690f1a0e6c69 }, 1392 :intvector{ -1,-1,-1,-1,0,1,1,1,2,2,3,3,3 }, 1393 :int{0}, :int{1}, "", "?", "\udc01" 1394 } 1395 { 1396 "iso-2022-kr", 1397 "\x61\u4e00\udc01\u4e00", 1398 :bin{ 1b242943610e6c690f1a0e6c690f }, 1399 :intvector{ -1,-1,-1,-1,0,1,1,1,2,2,3,3,3,3 }, 1400 :int{1}, :int{1}, "", "?", "" 1401 } 1402 1403 // ISO-2022-KR 1404 { 1405 "ibm-25546", 1406 "AB\uc88b\U00050005\uacccC", 1407 :bin{ 1b24294341420e41410f7b552b35303030357d0e306a0f43 }, 1408 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,3,3,3,3,3,3,3,3,5,5,5,6,6 }, 1409 :int{1}, :int{1}, "", "&U", "" 1410 } 1411 { 1412 "ibm-25546", 1413 "AB\uc88b\U00050005\uacccC", 1414 :bin{ 1b24294341420e41410f1a0e306a0f43 }, 1415 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5,6,6 }, 1416 :int{1}, :int{1}, "", "?\x00\x1a", "" 1417 } 1418 { 1419 "ibm-25546", 1420 "AB\uc88b\U00050005\uacccC", 1421 :bin{ 1b24294341420e41412f7e306a0f43 }, 1422 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,6,6 }, 1423 :int{1}, :int{1}, "", "?", "" 1424 } 1425 { 1426 "ibm-25546", 1427 "AB\uc88b\U00050005\uaccc", 1428 :bin{ 1b24294341420e41412f7e306a0f }, 1429 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5 }, 1430 :int{1}, :int{1}, "", "?", "" 1431 } 1432 { 1433 "ISO-2022-KR", 1434 "AB\uc88b\U00050005\uacccC", 1435 :bin{ 1b24294341420e41410f7b552b35303030357d0e306a0f43 }, 1436 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,3,3,3,3,3,3,3,3,5,5,5,6,6 }, 1437 :int{1}, :int{1}, "", "&U", "" 1438 } 1439 { 1440 "ISO-2022-KR", 1441 "AB\uc88b\U00050005\uacccC", 1442 :bin{ 1b24294341420e41410f1a0e306a0f43 }, 1443 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5,6,6 }, 1444 :int{1}, :int{1}, "", "?", "" 1445 } 1446 { 1447 "ISO-2022-KR", 1448 "AB\uc88b\U00050005\uacccC", 1449 :bin{ 1b24294341420e41412f7e306a0f43 }, 1450 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,6,6 }, 1451 :int{1}, :int{1}, "", "?\x00\x2f\x7e", "" 1452 } 1453 { 1454 "ISO-2022-KR", 1455 "AB\uc88b\U00050005\uaccc", 1456 :bin{ 1b24294341420e41412f7e306a0f }, 1457 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,5,5,5 }, 1458 :int{1}, :int{1}, "", "?\x00\x2f\x7e", "" 1459 } 1460 // ISO-2022-KR 1461 { 1462 "ibm-25546", 1463 "AB\uc88b\U00050005\uacccC", 1464 :bin{ 1b24294341420e41410f5c3530303035200e306a0f43 }, 1465 :intvector{ -1,-1,-1,-1,0,1,2,2,2,3,3,3,3,3,3,3,3,5,5,5,6,6 }, 1466 :int{1}, :int{1}, "", "&S", "" 1467 } 1468 1469 // ISO-2022-JP-2 with G2 designator & SS2 shift 1470 { 1471 "ISO-2022-JP-2", 1472 "CF\u758f\u038f\u7591", 1473 :bin{ 43461b244241411b2e461b4e3f353f1b2842 }, 1474 :intvector{ 0,1,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4 }, 1475 :int{1}, :int{1}, "", ".", "" 1476 } 1477 // JIS7 with Katakana 1478 { 1479 "JIS7", 1480 "AB\uff81\uff82AB", 1481 :bin{ 41420e41420f4142 }, 1482 :intvector{ 0,1,2,2,3,4,4,5 }, 1483 :int{1}, :int{1}, "", ".", "" 1484 } 1485 // JIS7 with shift to ASCII at the very end 1486 { 1487 "JIS7", 1488 "AB\uff81\uff82", 1489 :bin{ 41420e41420f }, 1490 :intvector{ 0,1,2,2,3,3 }, 1491 :int{1}, :int{1}, "", ".", "" 1492 } 1493 // JIS8 with Katakana 1494 { 1495 "JIS8", 1496 "A\uff81\\\xa5\uff82B", 1497 :bin{ 41c15c1b284a5cc2421b2842 }, 1498 :intvector{ 0,1,2,3,3,3,3,4,5,5,5,5 }, 1499 :int{1}, :int{1}, "", ".", "" 1500 } 1501 1502 /* 1503 * ICU 4.4 (ticket #7314) removes mappings for CNS 11643 planes 3..7 1504 // ISO-2022-CN-EXT with all subcharsets and shifts and with supplementary code points 1505 { 1506 "ISO-2022-CN-EXT", 1507 "\u4eae\u9f82\u56cd\u56cc\U0002a6d6\x30", 1508 :bin{ 1b2429410e41411b2429457e7c1b242a481b4e70341b2429477c341b242b4d1b4f664c0f30 }, 1509 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4,6,6 }, 1510 :int{1}, :int{1}, "", ".", "" 1511 } 1512 // ISO-2022-CN-EXT with shift to ASCII at the very end 1513 { 1514 "ISO-2022-CN-EXT", 1515 "\u4eae\u9f82\u56cd\u56cc\U0002a6d6", 1516 :bin{ 1b2429410e41411b2429457e7c1b242a481b4e70341b2429477c341b242b4d1b4f664c0f }, 1517 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4,4 }, 1518 :int{1}, :int{1}, "", ".", "" 1519 } 1520 // ISO-2022-CN-EXT without flush so do not shift to ASCII at the very end 1521 { 1522 "ISO-2022-CN-EXT", 1523 "\u4eae\u9f82\u56cd\u56cc\U0002a6d6", 1524 :bin{ 1b2429410e41411b2429457e7c1b242a481b4e70341b2429477c341b242b4d1b4f664c }, 1525 :intvector{ 0,0,0,0,0,0,0,1,1,1,1,1,1,2,2,2,2,2,2,2,2,3,3,3,3,3,3,4,4,4,4,4,4,4,4 }, 1526 :int{0}, :int{1}, "", ".", "" 1527 } 1528 */ 1529 1530 // windows-936 vs. ibm-1386 1531 { 1532 "ibm-1386", 1533 "\x1a\u20ac\u5555\x80\x81\U00055555", 1534 :bin{ 7fa2e3dffb7f7fa1a1 }, 1535 :intvector{ 0, 1, 1, 2, 2, 3, 4, 5, 5 }, 1536 :int{1}, :int{1}, "", "?", "" 1537 } 1538 { 1539 "windows-936", 1540 "\x1a\u20ac\u5555\x80\x81\U00055555", 1541 :bin{ 1a80dffb3f3f3f }, 1542 :intvector{ 0, 1, 2, 2, 3, 4, 5 }, 1543 :int{1}, :int{1}, "", "?", "" 1544 } 1545 1546 // verify that if a conversion table does not have any mapping for U+0000, 1547 // then there will not even be a phantom fallback to 00 1548 { 1549 "ibm-971", 1550 "\x00", 1551 :bin{ affe }, 1552 :intvector{ 0, 0 }, 1553 :int{1}, :int{1}, "", "?", "" 1554 } 1555 1556 { 1557 "*test4", 1558 "\x00", 1559 :bin{ ff }, 1560 :intvector{ 0 }, 1561 :int{1}, :int{1}, "", "?", "" 1562 } 1563 1564 // extension in testdata 1565 { 1566 "*test4x", 1567 "\u20ac\x09", 1568 :bin{ 0009 }, 1569 :intvector{ 0, 1 }, 1570 :int{1}, :int{1}, "", "?", "" 1571 } 1572 1573 // DBCS-only extensions 1574 { 1575 "ibm-970", 1576 "\x61\uffa1\u2015\ub000", 1577 :bin{ 611aa1aab2eb }, 1578 :intvector{ 0, 1, 2, 2, 3, 3 }, 1579 :int{1}, :int{1}, "", "?", "" 1580 } 1581 1582 { 1583 "ibm-971", 1584 "\x61\uffa1\u2015\ub000", 1585 :bin{ affeaffeaffeb2eb }, 1586 :intvector{ 0, 0, 1, 1, 2, 2, 3, 3 }, 1587 :int{1}, :int{1}, "", "?", "" 1588 } 1589 1590 { 1591 "ibm-1390,swaplfnl", 1592 "\uff63\u30C8\u30C8\u309A\u3000\x41\u20ac\x0a", 1593 :bin{ 430e4395ecc140400fc1e115 }, 1594 :intvector{ 0, 1, 1, 1, 2, 2, 4, 4, 5, 5, 6, 7 }, 1595 :int{1}, :int{0}, "", "?", "" 1596 } 1597 1598 { 1599 "ibm-16684", 1600 "\uff63\u30C8\u30C8\u309A\u3000\x41\u20ac\x0a", 1601 :bin{ fefe4395ecc14040fefe42e1fefe }, 1602 :intvector{ 0, 0, 1, 1, 2, 2, 4, 4, 5, 5, 6, 6, 7, 7 }, 1603 :int{1}, :int{0}, "", "?", "" 1604 } 1605 1606 { 1607 "ibm-1399", 1608 "\uff63\u30C8\u30C8\u309A\u3000\x41\u20ac\x0a", 1609 :bin{ 440e4395ecc140400fc1e125 }, 1610 :intvector{ 0, 1, 1, 1, 2, 2, 4, 4, 5, 5, 6, 7 }, 1611 :int{1}, :int{0}, "", "?", "" 1612 } 1613 1614 // <subchar1> from |2 mappings 1615 { 1616 "ibm-1390", 1617 "\x0e\x0f\u0901\U00050000\uffe8\uffee", 1618 :bin{ 3f3f0efefefefe0f3f3f }, 1619 :intvector{ 0, 1, 2, 2, 2, 3, 3, 5, 5, 6 }, 1620 :int{1}, :int{1}, "", "?", "" 1621 } 1622 1623 // <subchar1> from |2 mappings, and also contains a fallback to 00 1624 { 1625 "*test4", 1626 "\u20ac\u20ad\U00050005\U00023456\U0010ffff\x30", 1627 :bin{ 0000e10102030affff }, 1628 :intvector{ 0, 1, 2, 4, 4, 4, 4, 6, 8 }, 1629 :int{1}, :int{1}, "", "?", "" 1630 } 1631 1632 // setting a <subchar> resets the <subchar1> 1633 { 1634 "*test4", 1635 "\u20ac\u20ad\U00050005\U00023456\U0010ffff\x30", 1636 :bin{ 00000102030f0102030a0102030f0102030f }, 1637 :intvector{ 0, 1, 2, 2, 2, 2, 4, 4, 4, 4, 6, 6, 6, 6, 8, 8, 8, 8 }, 1638 :int{1}, :int{1}, "", "?\x00\x01\x02\x03\x0f", "" 1639 } 1640 1641 // fallback to 00 with old single-byte data structure 1642 { 1643 "*test1", 1644 "\u20ac\u20ad\U00101234\U00050000", 1645 :bin{ 000007ff }, 1646 :intvector{ 0, 1, 2, 4 }, 1647 :int{1}, :int{1}, "", "?", "" 1648 } 1649 1650 // extensions 1651 { 1652 "ibm-1390", 1653 "\u025a\u025a\u0300\u025a\u0301\u025a\u0302\uffe8\U0002a0f9", 1654 :bin{ 0ed896eccaeccbd896ea530f3f0eb7c20f }, 1655 :intvector{ 0, 0, 0, 1, 1, 3, 3, 5, 5, 6, 6, 7, 7, 8, 8, 8, 8 }, 1656 :int{1}, :int{0}, "", "?", "" 1657 } 1658 1659 { 1660 "*test3", 1661 "\xc4\xc4\xc4\U00101234\xc4\xc4\U00101234\x05", 1662 :bin{ ffffff070501020c }, 1663 :intvector{ 0, 1, 2, 3, 5, 5, 5, 5 }, 1664 :int{1}, :int{0}, "", "?", "" 1665 } 1666 1667 { 1668 "*test3", 1669 "\U00101234\U00101234\U00050005\U00101234\U00050005\U00060006", 1670 :bin{ 07070001020e05070001020f09 }, 1671 :intvector{ 0, 2, 2, 2, 2, 2, 2, 6, 6, 6, 6, 6, 6 }, 1672 :int{1}, :int{0}, "", "?", "" 1673 } 1674 1675 // normal conversions 1676 { 1677 "UTF-16LE", 1678 "1\U00010001\U000500022\ud8003\udc014", 1679 :bin{ 310000d801dc00d902dc3200fdff3300fdff3400 }, 1680 :intvector{ 0, 0, 1, 1, 1, 1, 3, 3, 3, 3, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9 }, 1681 :int{1}, :int{0}, "", "?", "" 1682 } 1683 { "UTF-16LE", "\ud800", :bin{""}, :intvector{}, :int{1}, :int{0}, "truncated", ".", "\ud800" } 1684 1685 { 1686 "UTF-16BE", 1687 "1\U00010001\U000500022\ud8003\udc014", 1688 :bin{ 0031d800dc01d900dc020032fffd0033fffd0034 }, 1689 :intvector{ 0, 0, 1, 1, 1, 1, 3, 3, 3, 3, 5, 5, 6, 6, 7, 7, 8, 8, 9, 9 }, 1690 :int{1}, :int{0}, "", "?", "" 1691 } 1692 { "UTF-16BE", "\ud800", :bin{""}, :intvector{}, :int{1}, :int{0}, "truncated", ".", "\ud800" } 1693 1694 { 1695 "SCSU", 1696 "1\U00010001\u00082\ud8003\udc014\ue001", 1697 :bin{ 310be000810108320efffd330efffd34186881 }, 1698 :intvector{ 0, 1, 1, 1, 1, 3, 3, 4, 5, 5, 5, 6, 7, 7, 7, 8, 9, 9, 9 }, 1699 :int{1}, :int{0}, "", "?", "" 1700 } 1701 { 1702 "x11-compound-text", 1703 "\u54A1\u00A5\u00E3\u0120\u0121\u011E\u0041\u0135\u02D9\u0E3F\u0100\u0157\u0384\u0660\u05D0\u0401", 1704 :bin{ 1b242944b5ac1b2d41a5e31b2d43d5f51b2d4dd01b2d41411b2d43bc1b2d42ff1b2d54df1b2d44c0b31b2d46b41b2d47b01b2d48e01b2d4ca1 }, 1705 :intvector{ }, 1706 :int{1}, :int{0}, "", "?", "" 1707 } 1708 // Test Gurmukhi (Bindi Tippi and Consonant clusters) 1709 { 1710 "iscii-gur", 1711 "\u0a15\u0a70\u0a02\u0a71\u0a15\u0a5c\u0a4d\u0a39\u000a\u0043\u0041\u000a", 1712 :bin { ef4bb3a2a2b3e8b3bfe9e8d80aef4b43410a }, 1713 :intvector{ 0, 0, 0, 1, 2, 3, 3, 3, 5, 5, 6, 7, 8, 8, 8, 9, 10, 11 }, 1714 :int{1}, :int{0}, "", ".", "" 1715 } 1716 // escape callback 1717 { 1718 "iscii-dev", 1719 "A\u0901\U00023456\u0902B\U00023456C", 1720 :bin{ 41ef42a1255544383444255544433536a24225554438344425554443353643 }, 1721 :intvector{ 1722 0, 1723 1,1,1, 1724 2,2,2,2,2,2, 1725 2,2,2,2,2,2, 1726 4, 1727 5, 1728 6,6,6,6,6,6, 1729 6,6,6,6,6,6, 1730 8 1731 }, 1732 :int{1}, :int{0}, "", "&", "" 1733 } 1734 1735 // escape callback (hex) 1736 { 1737 "iso-2022-jp", 1738 "\u3000\U00023456\u3001\U00023456B\u901c", 1739 :bin{ 1b244221211b284226237832333435363b1b244221221b284226237832333435363b42262378393031433b }, 1740 :intvector{ 1741 0,0,0,0,0, 1742 1,1,1,1,1,1,1,1,1,1,1,1, 1743 3,3,3,3,3, 1744 4,4,4,4,4,4,4,4,4,4,4,4, 1745 6, 1746 7,7,7,7,7,7,7,7 1747 }, 1748 :int{1}, :int{0}, "", "&X", "" 1749 } 1750 1751 // sub callback 1752 { 1753 "gb18030", 1754 "$\x7f\x80\u01f9\u20ac\u4e00\u9fa6\uffff\U00010000\U0010ffff", 1755 :bin{ 247f81308130a8bfa2e3d2bb82358f338431a43990308130e3329a35 }, 1756 :intvector{ 0, 1, 2, 2, 2, 2, 3, 3, 4, 4, 5, 5, 6, 6, 6, 6, 7, 7, 7, 7, 8, 8, 8, 8, 10, 10, 10, 10 }, 1757 :int{1}, :int{0}, "", "?", "" 1758 } 1759 1760 // skip callback 1761 { "ibm-930", "\u6D63\u6D64\u6D65\u6D66", :bin{ 0e5d5f5d63466b0f }, :intvector{ 0, 0, 0, 1, 1, 3, 3, 3 }, :int{1}, :int{0}, "", "0", "" } 1762 { "ibm-930", "\u6D63\u6D64\ud89a\u6D66", :bin{ 0e5d5f5d63466b0f }, :intvector{ 0, 0, 0, 1, 1, 3, 3, 3 }, :int{1}, :int{0}, "", "0", "" } 1763 { "ibm-930", "\u6D63\u6D64\ud89a\u6D66", :bin{ 0e5d5f5d63 }, :intvector{ 0, 0, 0, 1, 1 }, :int{1}, :int{0}, "illegal", "0i", "\ud89a" } 1764 1765 // sub callback for supplementary code point 1766 { "LATIN1", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{1}, :int{0}, "", "", "" } 1767 { "ibm-920", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{1}, :int{0}, "", "", "" } 1768 1769 // sub callback with AA as subchar 1770 { "ibm-920", "1\U000104012", :bin{ 31AA32 }, :intvector{ 0, 1, 3 }, :int{1}, :int{0}, "", "?\x00\xAA", "" } 1771 1772 // same but not flushing 1773 { "LATIN1", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{0}, :int{0}, "", "", "\U00010401" } 1774 { "ibm-920", "1\U000104012", :bin{ 311a32 }, :intvector{ 0, 1, 3 }, :int{0}, :int{0}, "", "", "\U00010401" } 1775 1776 // simple sample, no error handling 1777 { "UTF-8", "a\U0010FFFF", :bin{ 61F48FBFBF }, :intvector{ 0, 1, 1, 1, 1 }, :int{1}, :int{0}, "", "", "" } 1778 1779 // Verify that incomplete surrogates are handled as an error 1780 { "UTF-8", "a\udc00", :bin{ 61efbfbd }, :intvector{ 0, 1, 1, 1 }, :int{1}, :int{0}, "", "", "" } 1781 { "UTF-8", "a\ud800", :bin{ 61efbfbd }, :intvector{ 0, 1, 1, 1 }, :int{1}, :int{0}, "", "", "" } 1782 { "UTF-8", "a\udc00b", :bin{ 61efbfbd62 }, :intvector{ 0, 1, 1, 1, 2 }, :int{1}, :int{0}, "", "", "" } 1783 { "UTF-8", "a\ud800b", :bin{ 61efbfbd62 }, :intvector{ 0, 1, 1, 1, 2 }, :int{1}, :int{0}, "", "", "" } 1784 1785 // Code coverage for the EUC variants. 1786 { "IBM-eucJP", "\u0061\u4edd\u5bec\ud801\udc01\ud801\u0061\u00a2", :bin{ 61a1b88ff4ae618ee0 }, :intvector{ 0, 1, 1, 2, 2, 2, 6, 7, 7 }, :int{1}, :int{0}, "", "0", "" } 1787 { "IBM-eucJP", "\u0061\u4edd\u5bec\ud801\udc01\ud801\u0061\u00a2", :bin{ 61a1b88ff4aef4fef4fe618ee0 }, :intvector{ 0, 1, 1, 2, 2, 2, 3, 3, 5, 5, 6, 7, 7 }, :int{1}, :int{0}, "", "", "" } 1788 { "EUC-TW", "\u0061\u2295\u5BF2\ud801\udc01\ud801\u0061\u8706\u008a", :bin{ 61a2d38ea2dce561e6ca8a }, :intvector{ 0, 1, 1, 2, 2, 2, 2, 6, 7, 7, 8 }, :int{1}, :int{0}, "", "0", "" } 1789 { "EUC-TW", "\u0061\u2295\u5BF2\ud801\udc01\ud801\u0061\u8706\u008a", :bin{ 61a2d38ea2dce5fdfefdfe61e6ca8a }, :intvector{ 0, 1, 1, 2, 2, 2, 2, 3, 3, 5, 5, 6, 7, 7, 8 }, :int{1}, :int{0}, "", "", "" } 1790 1791 // Code Coverage for BOCU-1 1792 { 1793 "BOCU-1", 1794 "\u0041\ud841\ud888\udc81", 1795 :bin{ 91fbc555fd6349 }, 1796 :intvector{ 0, 1, 1, 1, 2, 2, 2 }, 1797 :int{1}, :int{0}, "", ".", "" 1798 } 1799 { 1800 "BOCU-1", 1801 "\ufe88\ufe70\udbff\udfff\u0061", 1802 :bin{ fbeda44ff0fe189bb821f05926 }, 1803 :intvector{}, 1804 :int{1}, :int{0}, "", ".", "" 1805 } 1806 // Improve code coverage for SCSU 1807 { 1808 "SCSU", 1809 "\ud899\udc7f\ud977\ud888\udc99\ud888\u0041", 1810 :bin{ 0fd899dc7fd888dc99e041 }, 1811 :intvector{}, 1812 :int{1}, :int{0}, "", "0", "" 1813 } 1814 { 1815 "gb18030", 1816 "\U00020087\ue790\ue78f\u1e3f", 1817 :bin{ 95329031a6dca6dba8bc }, 1818 :intvector{ 0,0,0,0,2,2,3,3,4,4 }, 1819 :int{1}, :int{0}, "", "0", "" 1820 } 1821 { 1822 "UTF-7", 1823 "\u00a3I\u00a3\u00a4", 1824 :bin{ 2b414b4d2d492b414b4d4170412d }, 1825 :intvector{ 0,0,0,0,0,1,2,2,2,3,3,3,3,3 }, 1826 :int{1}, :int{0}, "", "0", "" 1827 } 1828 // Bug #9601 direct-from-UTF-8 m:n Unicode:charset conversion. 1829 { 1830 "*test1bmp", 1831 "uv", 1832 :bin{ 08 }, 1833 :intvector{ 0 }, 1834 :int{1}, :int{0}, "", "?", "" 1835 } 1836 { 1837 "*test2", 1838 "\U00101234\U00050005", 1839 :bin{ 0700010e05 }, 1840 :intvector{ 0,0,0,0,0 }, 1841 :int{1}, :int{0}, "", "?", "" 1842 } 1843 } 1844 } 1845 1846 getUnicodeSet { 1847 // charset - will be opened, and ucnv_getUnicodeSet() called on it 1848 // map - set of code points and strings that must be in the returned set 1849 // mapnot - set of code points and strings that must *not* be in the returned set 1850 // which - numeric UConverterUnicodeSet value 1851 Headers { "charset", "map", "mapnot", "which" } 1852 Cases { 1853 // Test ticket 9602: Add "good one-way" mapping type (|4). 1854 // Excluded from roundtrip set, included in the set with fallbacks. 1855 { 1856 "+*test3", 1857 "[{#\uFE0F}]", 1858 "[#{#\uFE0E}]", 1859 :int{0} 1860 } 1861 { 1862 "+*test3", 1863 "[#{#\uFE0E}{#\uFE0F}]", 1864 "[]", 1865 :int{1} 1866 } 1867 // Unicode charsets that do not map surrogate code points 1868 { 1869 "UTF-8", 1870 "[\x00-\ud7ff\ue000-\U0010ffff]", 1871 "[\ud800-\udfff]", 1872 :int{0} 1873 } 1874 { 1875 "UTF-16", 1876 "[\x00-\ud7ff\ue000-\U0010ffff]", 1877 "[\ud800-\udfff]", 1878 :int{0} 1879 } 1880 { 1881 "UTF-16BE", 1882 "[\x00-\ud7ff\ue000-\U0010ffff]", 1883 "[\ud800-\udfff]", 1884 :int{0} 1885 } 1886 { 1887 "UTF-16LE", 1888 "[\x00-\ud7ff\ue000-\U0010ffff]", 1889 "[\ud800-\udfff]", 1890 :int{0} 1891 } 1892 { 1893 "UTF-32", 1894 "[\x00-\ud7ff\ue000-\U0010ffff]", 1895 "[\ud800-\udfff]", 1896 :int{0} 1897 } 1898 { 1899 "UTF-32BE", 1900 "[\x00-\ud7ff\ue000-\U0010ffff]", 1901 "[\ud800-\udfff]", 1902 :int{0} 1903 } 1904 { 1905 "UTF-32LE", 1906 "[\x00-\ud7ff\ue000-\U0010ffff]", 1907 "[\ud800-\udfff]", 1908 :int{0} 1909 } 1910 1911 // Unicode charsets that do map surrogate code points 1912 { 1913 "UTF-7", 1914 "[\x00-\U0010ffff]", 1915 "[]", 1916 :int{0} 1917 } 1918 { 1919 "SCSU", 1920 "[\x00-\U0010ffff]", 1921 "[]", 1922 :int{0} 1923 } 1924 { 1925 "BOCU-1", 1926 "[\x00-\U0010ffff]", 1927 "[]", 1928 :int{0} 1929 } 1930 { 1931 "CESU-8", 1932 "[\x00-\U0010ffff]", 1933 "[]", 1934 :int{0} 1935 } 1936 1937 // versions of ISO-2022-KR 1938 { 1939 "ISO-2022-KR", 1940 "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa1\xa4\xfe\u0111\u4e00\u4e01\uac00-\uac02\uffe6]", 1941 "[\x0e\x0f\x1b\x80-\xa0\xa3\xa5\xff-\u0110\uac03\uffe7-\U0010ffff]", 1942 :int{0} 1943 } 1944 { 1945 "ibm-25546", 1946 "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa1\xa4\xfe\u0111\u4e00\u4e01\uac00-\uac01\uffe6]", 1947 "[\x0e\x0f\x1b\x80-\xa0\xa3\xa5\xff-\u0110\uac02\uffe7-\U0010ffff]", 1948 :int{0} 1949 } 1950 1951 // versions of ISO-2022-JP 1952 // Start of Google Specific patches for SoftBank Shift JIS tables 1953 { 1954 "ISO-2022-JP", 1955 "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa5\u0391-\u03a1\u2015\u203e\u4e00\u4e01\uffe5]", 1956 "[\x0e\x0f\x1b\u0100-\u0113\u0385-\u038a\u2014\u301c\u4e02\u4e27\u4e29\u4fe0\u9eb5\uff61-\uff9f\uffe6-\U0010ffff]", 1957 :int{0} 1958 } 1959 { 1960 "ISO-2022-JP-2", 1961 "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa0-\u0113\u0384-\u0386\u0388-\u038a\u0390-\u03a1\u203e\u4e00-\u4e05\u4fe0\u663b\uffe6]", 1962 "[\x0e\x0f\x1b\uff61-\uff9f\uffe7-\U0010ffff]", 1963 :int{0} 1964 } 1965 { 1966 "JIS7", 1967 "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa0-\u0113\u0384-\u0386\u0388-\u038a\u0390-\u03a1\u203e\u4e00-\u4e05\u4fe0\u663b\uff61-\uff9f\uffe4\uffe6]", 1968 "[\x0e\x0f\x1b\uffe7-\U0010ffff]", 1969 :int{0} 1970 } 1971 // with fallbacks 1972 { 1973 "ISO-2022-JP", 1974 "[\x00-\x0d\x10-\x1a\x1c-\x7f\xa5\u0391-\u03a1\u2014\u2015\u203e\u301c\u4e00\u4e01\uff61-\uff9f\uffe5]", 1975 "[\x0e\x0f\x1b\xa6\u0100-\u0113\u0385-\u038a\u4e02\u4e27\u4e29\u4fe0\u9eb5\uffe6-\U0010ffff]", 1976 :int{1} 1977 } 1978 1979 // versions of ISO-2022-CN 1980 { 1981 "ISO-2022-CN", 1982 "[\x00-\x0d\x10-\x1a\x1c-\x7f\u4e00\u4e01\u9f98\ufe6b]", 1983 "[\x0e\x0f\x1b\u4e29\uffe6-\U0010ffff]", 1984 :int{0} 1985 } 1986 /* 1987 * ICU 4.4 (ticket #7314) removes mappings for CNS 11643 planes 3..7 1988 { 1989 "ISO-2022-CN-EXT", 1990 "[\x00-\x0d\x10-\x1a\x1c-\x7f\u4e00-\u4e05\u9f98\ufe6b\u4e28-\u4e2b\U00020000\U00020003-\U00020005\U00029664]", 1991 "[\x0e\x0f\x1b\U00020001\U00020002\U0002a6d7-\U0010ffff]", 1992 :int{0} 1993 } 1994 */ 1995 1996 // HZ 1997 { 1998 "HZ", 1999 "[\u0410-\u044f\u4e00\u4e01\u4e03]", 2000 "[\u4e02\u4e04-\u4e06\uac00-\ud7ff]", 2001 :int{0} 2002 } 2003 2004 // LMBCS 2005 { 2006 "LMBCS", 2007 "[\x00-\U0010ffff]", 2008 "[]", 2009 :int{0} 2010 } 2011 2012 // ISCII,version=0 (note: all versions of ISCII generates the same Unicodeset) 2013 { 2014 "iscii-dev", 2015 "[\x00-\xa0\u0909-\u0939\u0993-\u09a8\u0a13-\u0a28\u0a93-\u0aa8\u0ae6-\u0aef\u0b05-\u0b0c\u0b13-\u0b28\u0bae-\u0bb5\u0c12-\u0c28\u0c92-\u0ca8\u0d12-\u0d28]", 2016 "[\u0971-\u0975\u09e4\u0a4e-\u0a58\u0a80\u0b72-\u0b7a\u0bfb-\u0bff\u0c70-\u0c7f\u0c80\u0d00]", 2017 :int{0} 2018 } 2019 2020 { 2021 "iso-8859-1", 2022 "[\x00-\xff]", 2023 "[\u0100-\u01ff]", 2024 :int{0} 2025 } 2026 2027 { 2028 "us-ascii", 2029 "[\x00-\x7f]", 2030 "[\u0100-\u01ff]", 2031 :int{0} 2032 } 2033 // DBCS-only 2034 { 2035 "ibm-971", 2036 "[\xa1\xa4\uac01\ub000]", 2037 "[\x00-\x9f\u2015]", 2038 :int{0} 2039 } 2040 2041 { 2042 "ibm-16684", 2043 "[\xa0\xa1\xa4\xa6-\xab\xad-\u017f\u0254\u309b-\u30ff\u4e00-\u4e05\U00023d00\U000243bc\U0002a6b2" 2044 "{\u0254\u0300}{\u0254\u0301}{\u304b\u309a}{\u30ad\u309a}{\u30af\u309a}]", 2045 "[\x00-0x9f\xa2\xa3\xa5\xac\u0200-\u024f\U00010000-\U0001ffff\U0002a61b-\U0002a6b1]", 2046 :int{0} 2047 } 2048 2049 // extensions 2050 { 2051 "ibm-1390", 2052 "[\x00-\x0d\x10-\u017f\u0254\u309b-\u30ff\u4e00-\u4e05\U00023d00\U000243bc\U0002a6b2" 2053 "{\u0254\u0300}{\u0254\u0301}{\u304b\u309a}{\u30ad\u309a}{\u30af\u309a}]", 2054 "[\x0e\x0f\u0200-\u024f\U00010000-\U0001ffff\U0002a61b-\U0002a6b1]", 2055 :int{0} 2056 } 2057 2058 { 2059 "*test3", 2060 "[\x05\x0b\xc0\u20ac\U00023456\U00101234" 2061 "{\U00101234\U00050005\U00060006}{\U00101234\U00050005}{\U00101234\U00060006}{\xc4\xc4\U00101234\x05}]", 2062 "[\x06\x0e\U00034567\U000febcd{\U00101234\U00070007}]", 2063 :int{0} 2064 } 2065 } 2066 } 2067 } 2068} 2069