16f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 26f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org ********************************************************************** 36f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org * Copyright (C) 2005-2013, International Business Machines 46f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org * Corporation and others. All Rights Reserved. 56f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org ********************************************************************** 66f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org */ 76f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 86f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org#include "unicode/utypes.h" 96f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 106f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org#include "cmemory.h" 116f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 126f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org#if !UCONFIG_NO_CONVERSION 136f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org#include "csrsbcs.h" 146f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org#include "csmatch.h" 156f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 166f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org#define N_GRAM_SIZE 3 176f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org#define N_GRAM_MASK 0xFFFFFF 186f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org#define ARRAY_SIZE(array) (sizeof array / sizeof array[0]) 196f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 206f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgU_NAMESPACE_BEGIN 216f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 226f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgNGramParser::NGramParser(const int32_t *theNgramList, const uint8_t *theCharMap) 236f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org : ngram(0), byteIndex(0) 246f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 256f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org ngramList = theNgramList; 266f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org charMap = theCharMap; 276f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 286f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org ngramCount = hitCount = 0; 296f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 306f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 316f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 326f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org * Binary search for value in table, which must have exactly 64 entries. 336f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org */ 346f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 356f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgint32_t NGramParser::search(const int32_t *table, int32_t value) 366f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 376f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t index = 0; 386f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 396f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if (table[index + 32] <= value) { 406f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org index += 32; 416f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 426f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 436f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if (table[index + 16] <= value) { 446f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org index += 16; 456f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 466f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 476f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if (table[index + 8] <= value) { 486f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org index += 8; 496f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 506f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 516f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if (table[index + 4] <= value) { 526f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org index += 4; 536f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 546f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 556f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if (table[index + 2] <= value) { 566f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org index += 2; 576f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 586f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 596f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if (table[index + 1] <= value) { 606f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org index += 1; 616f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 626f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 636f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if (table[index] > value) { 646f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org index -= 1; 656f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 666f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 676f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if (index < 0 || table[index] != value) { 686f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return -1; 696f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 706f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 716f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return index; 726f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 736f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 746f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgvoid NGramParser::lookup(int32_t thisNgram) 756f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 766f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org ngramCount += 1; 776f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 786f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if (search(ngramList, thisNgram) >= 0) { 796f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org hitCount += 1; 806f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 816f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 826f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 836f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 846f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgvoid NGramParser::addByte(int32_t b) 856f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 866f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org ngram = ((ngram << 8) + b) & N_GRAM_MASK; 876f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org lookup(ngram); 886f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 896f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 906f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgint32_t NGramParser::nextByte(InputText *det) 916f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 926f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if (byteIndex >= det->fInputLen) { 936f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return -1; 946f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 956f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 966f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return det->fInputBytes[byteIndex++]; 976f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 986f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 996f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgvoid NGramParser::parseCharacters(InputText *det) 1006f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 1016f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t b; 1026f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org bool ignoreSpace = FALSE; 1036f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 1046f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org while ((b = nextByte(det)) >= 0) { 1056f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org uint8_t mb = charMap[b]; 1066f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 1076f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // TODO: 0x20 might not be a space in all character sets... 1086f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if (mb != 0) { 1096f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if (!(mb == 0x20 && ignoreSpace)) { 1106f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org addByte(mb); 1116f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 1126f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 1136f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org ignoreSpace = (mb == 0x20); 1146f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 1156f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 1166f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 1176f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 1186f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgint32_t NGramParser::parse(InputText *det) 1196f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 1206f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org parseCharacters(det); 1216f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 1226f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // TODO: Is this OK? The buffer could have ended in the middle of a word... 1236f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org addByte(0x20); 1246f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 1256f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org double rawPercent = (double) hitCount / (double) ngramCount; 1266f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 1276f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // if (rawPercent <= 2.0) { 1286f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // return 0; 1296f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // } 1306f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 1316f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // TODO - This is a bit of a hack to take care of a case 1326f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // were we were getting a confidence of 135... 1336f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if (rawPercent > 0.33) { 1346f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return 98; 1356f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 1366f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 1376f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return (int32_t) (rawPercent * 300.0); 1386f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 1396f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 1406f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const uint8_t unshapeMap_IBM420[] = { 1416f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -A -B -C -D -E -F */ 1426f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 0- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 1436f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 1- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 1446f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 2- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 1456f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 3- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 1466f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 4- */ 0x40, 0x40, 0x42, 0x42, 0x44, 0x45, 0x46, 0x47, 0x47, 0x49, 0x4A, 0x4B, 0x4C, 0x4D, 0x4E, 0x4F, 1476f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 5- */ 0x50, 0x49, 0x52, 0x53, 0x54, 0x55, 0x56, 0x56, 0x58, 0x58, 0x5A, 0x5B, 0x5C, 0x5D, 0x5E, 0x5F, 1486f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 6- */ 0x60, 0x61, 0x62, 0x63, 0x63, 0x65, 0x65, 0x67, 0x67, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 1496f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 7- */ 0x69, 0x71, 0x71, 0x73, 0x74, 0x75, 0x76, 0x77, 0x77, 0x79, 0x7A, 0x7B, 0x7C, 0x7D, 0x7E, 0x7F, 1506f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 8- */ 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x80, 0x8B, 0x8B, 0x8D, 0x8D, 0x8F, 1516f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 9- */ 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9A, 0x9A, 0x9A, 0x9E, 0x9E, 1526f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* A- */ 0x9E, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x9E, 0xAB, 0xAB, 0xAD, 0xAD, 0xAF, 1536f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* B- */ 0xAF, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xB1, 0xBB, 0xBB, 0xBD, 0xBD, 0xBF, 1546f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* C- */ 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, 0xBF, 0xCC, 0xBF, 0xCE, 0xCF, 1556f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* D- */ 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, 0xDA, 0xDA, 0xDC, 0xDC, 0xDC, 0xDF, 1566f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* E- */ 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 1576f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* F- */ 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 1586f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 1596f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 1606f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgNGramParser_IBM420::NGramParser_IBM420(const int32_t *theNgramList, const uint8_t *theCharMap):NGramParser(theNgramList, theCharMap) 1616f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 1626f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org alef = 0x00; 1636f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 1646f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 1656f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 1666f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgint32_t NGramParser_IBM420::isLamAlef(int32_t b) 1676f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 1686f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if(b == 0xB2 || b == 0xB3){ 1696f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return 0x47; 1706f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }else if(b == 0xB4 || b == 0xB5){ 1716f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return 0x49; 1726f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }else if(b == 0xB8 || b == 0xB9){ 1736f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return 0x56; 1746f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }else 1756f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return 0x00; 1766f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 1776f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 1786f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 1796f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org* Arabic shaping needs to be done manually. Cannot call ArabicShaping class 1806f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org* because CharsetDetector is dealing with bytes not Unicode code points. We could 1816f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org* convert the bytes to Unicode code points but that would leave us dependent 1826f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org* on CharsetICU which we try to avoid. IBM420 converter amongst different versions 1836f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org* of JDK can produce different results and therefore is also avoided. 1846f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org*/ 1856f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgint32_t NGramParser_IBM420::nextByte(InputText *det) 1866f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 1876f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 1886f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if (byteIndex >= det->fInputLen || det->fInputBytes[byteIndex] == 0) { 1896f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return -1; 1906f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 1916f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int next; 1926f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 1936f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org alef = isLamAlef(det->fInputBytes[byteIndex]); 1946f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if(alef != 0x00) 1956f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org next = 0xB1 & 0xFF; 1966f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org else 1976f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org next = unshapeMap_IBM420[det->fInputBytes[byteIndex]& 0xFF] & 0xFF; 1986f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 1996f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org byteIndex++; 2006f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 2016f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return next; 2026f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 2036f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 2046f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgvoid NGramParser_IBM420::parseCharacters(InputText *det) 2056f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 2066f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t b; 2076f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org bool ignoreSpace = FALSE; 2086f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 2096f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org while ((b = nextByte(det)) >= 0) { 2106f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org uint8_t mb = charMap[b]; 2116f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 2126f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // TODO: 0x20 might not be a space in all character sets... 2136f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if (mb != 0) { 2146f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if (!(mb == 0x20 && ignoreSpace)) { 2156f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org addByte(mb); 2166f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 2176f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org ignoreSpace = (mb == 0x20); 2186f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 2196f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 2206f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if(alef != 0x00){ 2216f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org mb = charMap[alef & 0xFF]; 2226f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 2236f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // TODO: 0x20 might not be a space in all character sets... 2246f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if (mb != 0) { 2256f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if (!(mb == 0x20 && ignoreSpace)) { 2266f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org addByte(mb); 2276f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 2286f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 2296f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org ignoreSpace = (mb == 0x20); 2306f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 2316f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 2326f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 2336f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 2346f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 2356f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 2366f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_sbcs::CharsetRecog_sbcs() 2376f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 2386f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing else to do 2396f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 2406f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 2416f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_sbcs::~CharsetRecog_sbcs() 2426f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 2436f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 2446f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 2456f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 2466f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgint32_t CharsetRecog_sbcs::match_sbcs(InputText *det, const int32_t ngrams[], const uint8_t byteMap[]) const 2476f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 2486f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org NGramParser parser(ngrams, byteMap); 2496f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t result; 2506f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 2516f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org result = parser.parse(det); 2526f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 2536f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return result; 2546f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 2556f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 2566f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const uint8_t charMap_8859_1[] = { 2576f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 2586f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 2596f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 2606f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 2616f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 2626f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 2636f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 2646f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 2656f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 2666f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 2676f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 2686f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 2696f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 2706f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 2716f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 2726f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 2736f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 2746f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 2756f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 2766f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 2776f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 2786f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20, 2796f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20, 2806f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0x20, 2816f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 2826f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 2836f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, 2846f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xDF, 2856f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 2866f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 2876f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, 2886f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 2896f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 2906f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 2916f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const uint8_t charMap_8859_2[] = { 2926f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 2936f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 2946f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 2956f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 2966f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 2976f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 2986f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 2996f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3006f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 3016f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 3026f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 3036f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 3046f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 3056f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 3066f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 3076f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 3086f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3096f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3106f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3116f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3126f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0xB1, 0x20, 0xB3, 0x20, 0xB5, 0xB6, 0x20, 3136f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0xB9, 0xBA, 0xBB, 0xBC, 0x20, 0xBE, 0xBF, 3146f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0xB1, 0x20, 0xB3, 0x20, 0xB5, 0xB6, 0xB7, 3156f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0xB9, 0xBA, 0xBB, 0xBC, 0x20, 0xBE, 0xBF, 3166f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 3176f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 3186f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, 3196f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xDF, 3206f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 3216f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 3226f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, 3236f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0x20, 3246f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 3256f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 3266f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const uint8_t charMap_8859_5[] = { 3276f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3286f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3296f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3306f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3316f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 3326f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3336f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3346f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3356f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 3366f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 3376f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 3386f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 3396f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 3406f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 3416f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 3426f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 3436f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3446f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3456f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3466f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3476f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 3486f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x20, 0xFE, 0xFF, 3496f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 3506f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 3516f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 3526f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 3536f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 3546f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 3556f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 3566f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 3576f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 3586f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x20, 0xFE, 0xFF, 3596f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 3606f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 3616f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const uint8_t charMap_8859_6[] = { 3626f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3636f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3646f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3656f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3666f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 3676f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3686f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3696f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3706f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 3716f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 3726f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 3736f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 3746f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 3756f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 3766f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 3776f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 3786f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3796f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3806f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3816f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3826f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3836f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3846f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3856f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3866f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 3876f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 3886f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 3896f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xD8, 0xD9, 0xDA, 0x20, 0x20, 0x20, 0x20, 0x20, 3906f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 3916f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE8, 0xE9, 0xEA, 0x20, 0x20, 0x20, 0x20, 0x20, 3926f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3936f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3946f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 3956f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 3966f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const uint8_t charMap_8859_7[] = { 3976f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3986f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 3996f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4006f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4016f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 4026f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4036f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4046f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4056f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 4066f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 4076f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 4086f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 4096f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 4106f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 4116f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 4126f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 4136f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4146f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4156f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4166f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4176f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0xA1, 0xA2, 0x20, 0x20, 0x20, 0x20, 0x20, 4186f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4196f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0xDC, 0x20, 4206f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xDD, 0xDE, 0xDF, 0x20, 0xFC, 0x20, 0xFD, 0xFE, 4216f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xC0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 4226f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 4236f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF0, 0xF1, 0x20, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 4246f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF8, 0xF9, 0xFA, 0xFB, 0xDC, 0xDD, 0xDE, 0xDF, 4256f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 4266f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 4276f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 4286f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0x20, 4296f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 4306f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 4316f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const uint8_t charMap_8859_8[] = { 4326f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4336f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4346f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4356f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4366f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 4376f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4386f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4396f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4406f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 4416f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 4426f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 4436f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 4446f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 4456f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 4466f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 4476f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 4486f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4496f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4506f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4516f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4526f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4536f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4546f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20, 4556f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4566f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4576f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4586f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4596f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4606f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 4616f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 4626f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 4636f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF8, 0xF9, 0xFA, 0x20, 0x20, 0x20, 0x20, 0x20, 4646f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 4656f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 4666f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const uint8_t charMap_8859_9[] = { 4676f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4686f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4696f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4706f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4716f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 4726f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4736f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4746f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4756f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 4766f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 4776f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 4786f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 4796f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 4806f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 4816f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 4826f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 4836f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4846f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4856f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4866f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4876f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 4886f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20, 4896f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20, 4906f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0x20, 4916f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 4926f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 4936f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, 4946f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0x69, 0xFE, 0xDF, 4956f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 4966f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 4976f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0x20, 4986f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 4996f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 5006f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 5016f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const int32_t ngrams_windows_1251[] = { 5026f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20E220, 0x20E2EE, 0x20E4EE, 0x20E7E0, 0x20E820, 0x20EAE0, 0x20EAEE, 0x20EDE0, 0x20EDE5, 0x20EEE1, 0x20EFEE, 0x20EFF0, 0x20F0E0, 0x20F1EE, 0x20F1F2, 0x20F2EE, 5036f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20F7F2, 0x20FDF2, 0xE0EDE8, 0xE0F2FC, 0xE3EE20, 0xE5EBFC, 0xE5EDE8, 0xE5F1F2, 0xE5F220, 0xE820EF, 0xE8E520, 0xE8E820, 0xE8FF20, 0xEBE5ED, 0xEBE820, 0xEBFCED, 5046f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xEDE020, 0xEDE520, 0xEDE8E5, 0xEDE8FF, 0xEDEE20, 0xEDEEE2, 0xEE20E2, 0xEE20EF, 0xEE20F1, 0xEEE220, 0xEEE2E0, 0xEEE3EE, 0xEEE920, 0xEEEBFC, 0xEEEC20, 0xEEF1F2, 5056f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xEFEEEB, 0xEFF0E5, 0xEFF0E8, 0xEFF0EE, 0xF0E0E2, 0xF0E5E4, 0xF1F2E0, 0xF1F2E2, 0xF1F2E8, 0xF1FF20, 0xF2E5EB, 0xF2EE20, 0xF2EEF0, 0xF2FC20, 0xF7F2EE, 0xFBF520, 5066f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 5076f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 5086f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const uint8_t charMap_windows_1251[] = { 5096f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5106f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5116f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5126f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5136f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 5146f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5156f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5166f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5176f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 5186f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 5196f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 5206f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 5216f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 5226f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 5236f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 5246f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 5256f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x90, 0x83, 0x20, 0x83, 0x20, 0x20, 0x20, 0x20, 5266f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x9A, 0x20, 0x9C, 0x9D, 0x9E, 0x9F, 5276f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5286f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x9A, 0x20, 0x9C, 0x9D, 0x9E, 0x9F, 5296f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0xA2, 0xA2, 0xBC, 0x20, 0xB4, 0x20, 0x20, 5306f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xB8, 0x20, 0xBA, 0x20, 0x20, 0x20, 0x20, 0xBF, 5316f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0xB3, 0xB3, 0xB4, 0xB5, 0x20, 0x20, 5326f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xB8, 0x20, 0xBA, 0x20, 0xBC, 0xBE, 0xBE, 0xBF, 5336f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 5346f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 5356f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 5366f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 5376f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 5386f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 5396f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, 5406f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, 5416f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 5426f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 5436f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const int32_t ngrams_windows_1256[] = { 5446f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20C7E1, 0x20C7E4, 0x20C8C7, 0x20DAE1, 0x20DDED, 0x20E1E1, 0x20E3E4, 0x20E6C7, 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E120, 0xC7E1C3, 0xC7E1C7, 0xC7E1C8, 5456f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xC7E1CA, 0xC7E1CC, 0xC7E1CD, 0xC7E1CF, 0xC7E1D3, 0xC7E1DA, 0xC7E1DE, 0xC7E1E3, 0xC7E1E6, 0xC7E1ED, 0xC7E320, 0xC7E420, 0xC7E4CA, 0xC820C7, 0xC920C7, 0xC920DD, 5466f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xC920E1, 0xC920E3, 0xC920E6, 0xCA20C7, 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920, 0xD320C7, 0xDA20C7, 0xDAE1EC, 0xDDED20, 0xE120C7, 0xE1C920, 0xE1EC20, 0xE1ED20, 5476f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE320C7, 0xE3C720, 0xE3C920, 0xE3E420, 0xE420C7, 0xE520C7, 0xE5C720, 0xE6C7E1, 0xE6E420, 0xEC20C7, 0xED20C7, 0xED20E3, 0xED20E6, 0xEDC920, 0xEDD120, 0xEDE420, 5486f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 5496f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 5506f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const uint8_t charMap_windows_1256[] = { 5516f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5526f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5536f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5546f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5556f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 5566f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5576f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5586f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5596f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 5606f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 5616f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 5626f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 5636f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 5646f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 5656f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 5666f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 5676f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x81, 0x20, 0x83, 0x20, 0x20, 0x20, 0x20, 5686f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x88, 0x20, 0x8A, 0x20, 0x9C, 0x8D, 0x8E, 0x8F, 5696f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x90, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5706f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x98, 0x20, 0x9A, 0x20, 0x9C, 0x20, 0x20, 0x9F, 5716f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5726f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0xAA, 0x20, 0x20, 0x20, 0x20, 0x20, 5736f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0xB5, 0x20, 0x20, 5746f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5756f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 5766f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 5776f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0x20, 5786f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 5796f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 5806f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE8, 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 5816f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0xF4, 0x20, 0x20, 0x20, 5826f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0xF9, 0x20, 0xFB, 0xFC, 0x20, 0x20, 0xFF, 5836f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 5846f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 5856f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const int32_t ngrams_KOI8_R[] = { 5866f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20C4CF, 0x20C920, 0x20CBC1, 0x20CBCF, 0x20CEC1, 0x20CEC5, 0x20CFC2, 0x20D0CF, 0x20D0D2, 0x20D2C1, 0x20D3CF, 0x20D3D4, 0x20D4CF, 0x20D720, 0x20D7CF, 0x20DAC1, 5876f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20DCD4, 0x20DED4, 0xC1CEC9, 0xC1D4D8, 0xC5CCD8, 0xC5CEC9, 0xC5D3D4, 0xC5D420, 0xC7CF20, 0xC920D0, 0xC9C520, 0xC9C920, 0xC9D120, 0xCCC5CE, 0xCCC920, 0xCCD8CE, 5886f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xCEC120, 0xCEC520, 0xCEC9C5, 0xCEC9D1, 0xCECF20, 0xCECFD7, 0xCF20D0, 0xCF20D3, 0xCF20D7, 0xCFC7CF, 0xCFCA20, 0xCFCCD8, 0xCFCD20, 0xCFD3D4, 0xCFD720, 0xCFD7C1, 5896f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xD0CFCC, 0xD0D2C5, 0xD0D2C9, 0xD0D2CF, 0xD2C1D7, 0xD2C5C4, 0xD3D120, 0xD3D4C1, 0xD3D4C9, 0xD3D4D7, 0xD4C5CC, 0xD4CF20, 0xD4CFD2, 0xD4D820, 0xD9C820, 0xDED4CF, 5906f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 5916f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 5926f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const uint8_t charMap_KOI8_R[] = { 5936f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5946f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5956f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5966f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5976f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x00, 5986f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 5996f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 6006f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 6016f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 6026f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 6036f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 6046f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 6056f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x61, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 6066f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x68, 0x69, 0x6A, 0x6B, 0x6C, 0x6D, 0x6E, 0x6F, 6076f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 6086f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x78, 0x79, 0x7A, 0x20, 0x20, 0x20, 0x20, 0x20, 6096f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 6106f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 6116f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 6126f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 6136f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0xA3, 0x20, 0x20, 0x20, 0x20, 6146f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 6156f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0xA3, 0x20, 0x20, 0x20, 0x20, 6166f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 0x20, 6176f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 6186f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 6196f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 6206f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 6216f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 6226f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xC8, 0xC9, 0xCA, 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 6236f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 6246f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xD8, 0xD9, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 6256f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 6266f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 6276f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const int32_t ngrams_IBM424_he_rtl[] = { 6286f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x404146, 0x404148, 0x404151, 0x404171, 0x404251, 0x404256, 0x404541, 0x404546, 0x404551, 0x404556, 0x404562, 0x404569, 0x404571, 0x405441, 0x405445, 0x405641, 6296f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x406254, 0x406954, 0x417140, 0x454041, 0x454042, 0x454045, 0x454054, 0x454056, 0x454069, 0x454641, 0x464140, 0x465540, 0x465740, 0x466840, 0x467140, 0x514045, 6306f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x514540, 0x514671, 0x515155, 0x515540, 0x515740, 0x516840, 0x517140, 0x544041, 0x544045, 0x544140, 0x544540, 0x554041, 0x554042, 0x554045, 0x554054, 0x554056, 6316f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x554069, 0x564540, 0x574045, 0x584540, 0x585140, 0x585155, 0x625440, 0x684045, 0x685155, 0x695440, 0x714041, 0x714042, 0x714045, 0x714054, 0x714056, 0x714069, 6326f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 6336f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 6346f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const int32_t ngrams_IBM424_he_ltr[] = { 6356f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x404146, 0x404154, 0x404551, 0x404554, 0x404556, 0x404558, 0x405158, 0x405462, 0x405469, 0x405546, 0x405551, 0x405746, 0x405751, 0x406846, 0x406851, 0x407141, 6366f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x407146, 0x407151, 0x414045, 0x414054, 0x414055, 0x414071, 0x414540, 0x414645, 0x415440, 0x415640, 0x424045, 0x424055, 0x424071, 0x454045, 0x454051, 0x454054, 6376f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x454055, 0x454057, 0x454068, 0x454071, 0x455440, 0x464140, 0x464540, 0x484140, 0x514140, 0x514240, 0x514540, 0x544045, 0x544055, 0x544071, 0x546240, 0x546940, 6386f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x555151, 0x555158, 0x555168, 0x564045, 0x564055, 0x564071, 0x564240, 0x564540, 0x624540, 0x694045, 0x694055, 0x694071, 0x694540, 0x714140, 0x714540, 0x714651, 6396f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 6406f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 6416f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const uint8_t charMap_IBM424_he[] = { 6426f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -A -B -C -D -E -F */ 6436f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 0- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6446f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 1- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6456f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 2- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6466f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 3- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6476f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 4- */ 0x40, 0x41, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6486f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 5- */ 0x40, 0x51, 0x52, 0x53, 0x54, 0x55, 0x56, 0x57, 0x58, 0x59, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6496f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 6- */ 0x40, 0x40, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6506f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 7- */ 0x40, 0x71, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x00, 0x40, 0x40, 6516f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 8- */ 0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6526f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 9- */ 0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6536f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* A- */ 0xA0, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6546f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* B- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6556f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* C- */ 0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6566f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* D- */ 0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6576f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* E- */ 0x40, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6586f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* F- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6596f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 6606f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 6616f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const int32_t ngrams_IBM420_ar_rtl[] = { 6626f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x4056B1, 0x4056BD, 0x405856, 0x409AB1, 0x40ABDC, 0x40B1B1, 0x40BBBD, 0x40CF56, 0x564056, 0x564640, 0x566340, 0x567540, 0x56B140, 0x56B149, 0x56B156, 0x56B158, 6636f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x56B163, 0x56B167, 0x56B169, 0x56B173, 0x56B178, 0x56B19A, 0x56B1AD, 0x56B1BB, 0x56B1CF, 0x56B1DC, 0x56BB40, 0x56BD40, 0x56BD63, 0x584056, 0x624056, 0x6240AB, 6646f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x6240B1, 0x6240BB, 0x6240CF, 0x634056, 0x734056, 0x736240, 0x754056, 0x756240, 0x784056, 0x9A4056, 0x9AB1DA, 0xABDC40, 0xB14056, 0xB16240, 0xB1DA40, 0xB1DC40, 6656f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xBB4056, 0xBB5640, 0xBB6240, 0xBBBD40, 0xBD4056, 0xBF4056, 0xBF5640, 0xCF56B1, 0xCFBD40, 0xDA4056, 0xDC4056, 0xDC40BB, 0xDC40CF, 0xDC6240, 0xDC7540, 0xDCBD40, 6666f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 6676f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 6686f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const int32_t ngrams_IBM420_ar_ltr[] = { 6696f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x404656, 0x4056BB, 0x4056BF, 0x406273, 0x406275, 0x4062B1, 0x4062BB, 0x4062DC, 0x406356, 0x407556, 0x4075DC, 0x40B156, 0x40BB56, 0x40BD56, 0x40BDBB, 0x40BDCF, 6706f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x40BDDC, 0x40DAB1, 0x40DCAB, 0x40DCB1, 0x49B156, 0x564056, 0x564058, 0x564062, 0x564063, 0x564073, 0x564075, 0x564078, 0x56409A, 0x5640B1, 0x5640BB, 0x5640BD, 6716f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x5640BF, 0x5640DA, 0x5640DC, 0x565840, 0x56B156, 0x56CF40, 0x58B156, 0x63B156, 0x63BD56, 0x67B156, 0x69B156, 0x73B156, 0x78B156, 0x9AB156, 0xAB4062, 0xADB156, 6726f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xB14062, 0xB15640, 0xB156CF, 0xB19A40, 0xB1B140, 0xBB4062, 0xBB40DC, 0xBBB156, 0xBD5640, 0xBDBB40, 0xCF4062, 0xCF40DC, 0xCFB156, 0xDAB19A, 0xDCAB40, 0xDCB156 6736f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 6746f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 6756f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const uint8_t charMap_IBM420_ar[]= { 6766f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* -0 -1 -2 -3 -4 -5 -6 -7 -8 -9 -A -B -C -D -E -F */ 6776f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 0- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6786f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 1- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6796f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 2- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6806f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 3- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6816f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 4- */ 0x40, 0x40, 0x42, 0x43, 0x44, 0x45, 0x46, 0x47, 0x48, 0x49, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6826f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 5- */ 0x40, 0x51, 0x52, 0x40, 0x40, 0x55, 0x56, 0x57, 0x58, 0x59, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6836f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 6- */ 0x40, 0x40, 0x62, 0x63, 0x64, 0x65, 0x66, 0x67, 0x68, 0x69, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6846f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 7- */ 0x70, 0x71, 0x72, 0x73, 0x74, 0x75, 0x76, 0x77, 0x78, 0x79, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 6856f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 8- */ 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, 0x8F, 6866f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* 9- */ 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, 0x9E, 0x9F, 6876f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* A- */ 0xA0, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, 0xAD, 0xAE, 0xAF, 6886f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* B- */ 0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0x40, 0x40, 0xB8, 0xB9, 0xBA, 0xBB, 0xBC, 0xBD, 0xBE, 0xBF, 6896f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* C- */ 0x40, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x40, 0xCB, 0x40, 0xCD, 0x40, 0xCF, 6906f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* D- */ 0x40, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 6916f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* E- */ 0x40, 0x40, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xEA, 0xEB, 0x40, 0xED, 0xEE, 0xEF, 6926f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org/* F- */ 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0x40, 0xFB, 0xFC, 0xFD, 0xFE, 0x40, 6936f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 6946f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 6956f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org//ISO-8859-1,2,5,6,7,8,9 Ngrams 6966f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 6976f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstruct NGramsPlusLang { 6986f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org const int32_t ngrams[64]; 6996f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org const char * lang; 7006f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 7016f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 7026f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const NGramsPlusLang ngrams_8859_1[] = { 7036f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7046f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7056f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x206120, 0x20616E, 0x206265, 0x20636F, 0x20666F, 0x206861, 0x206865, 0x20696E, 0x206D61, 0x206F66, 0x207072, 0x207265, 0x207361, 0x207374, 0x207468, 0x20746F, 7066f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x207768, 0x616964, 0x616C20, 0x616E20, 0x616E64, 0x617320, 0x617420, 0x617465, 0x617469, 0x642061, 0x642074, 0x652061, 0x652073, 0x652074, 0x656420, 0x656E74, 7076f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x657220, 0x657320, 0x666F72, 0x686174, 0x686520, 0x686572, 0x696420, 0x696E20, 0x696E67, 0x696F6E, 0x697320, 0x6E2061, 0x6E2074, 0x6E6420, 0x6E6720, 0x6E7420, 7086f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x6F6620, 0x6F6E20, 0x6F7220, 0x726520, 0x727320, 0x732061, 0x732074, 0x736169, 0x737420, 0x742074, 0x746572, 0x746861, 0x746865, 0x74696F, 0x746F20, 0x747320, 7096f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 7106f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org "en" 7116f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 7126f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7136f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7146f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x206166, 0x206174, 0x206465, 0x20656E, 0x206572, 0x20666F, 0x206861, 0x206920, 0x206D65, 0x206F67, 0x2070E5, 0x207369, 0x207374, 0x207469, 0x207669, 0x616620, 7156f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x616E20, 0x616E64, 0x617220, 0x617420, 0x646520, 0x64656E, 0x646572, 0x646574, 0x652073, 0x656420, 0x656465, 0x656E20, 0x656E64, 0x657220, 0x657265, 0x657320, 7166f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x657420, 0x666F72, 0x676520, 0x67656E, 0x676572, 0x696765, 0x696C20, 0x696E67, 0x6B6520, 0x6B6B65, 0x6C6572, 0x6C6967, 0x6C6C65, 0x6D6564, 0x6E6465, 0x6E6520, 7176f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x6E6720, 0x6E6765, 0x6F6720, 0x6F6D20, 0x6F7220, 0x70E520, 0x722064, 0x722065, 0x722073, 0x726520, 0x737465, 0x742073, 0x746520, 0x746572, 0x74696C, 0x766572, 7186f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 7196f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org "da" 7206f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 7216f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7226f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7236f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20616E, 0x206175, 0x206265, 0x206461, 0x206465, 0x206469, 0x206569, 0x206765, 0x206861, 0x20696E, 0x206D69, 0x207363, 0x207365, 0x20756E, 0x207665, 0x20766F, 7246f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x207765, 0x207A75, 0x626572, 0x636820, 0x636865, 0x636874, 0x646173, 0x64656E, 0x646572, 0x646965, 0x652064, 0x652073, 0x65696E, 0x656974, 0x656E20, 0x657220, 7256f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x657320, 0x67656E, 0x68656E, 0x687420, 0x696368, 0x696520, 0x696E20, 0x696E65, 0x697420, 0x6C6963, 0x6C6C65, 0x6E2061, 0x6E2064, 0x6E2073, 0x6E6420, 0x6E6465, 7266f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x6E6520, 0x6E6720, 0x6E6765, 0x6E7465, 0x722064, 0x726465, 0x726569, 0x736368, 0x737465, 0x742064, 0x746520, 0x74656E, 0x746572, 0x756E64, 0x756E67, 0x766572, 7276f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 7286f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org "de" 7296f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 7306f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7316f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7326f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x206120, 0x206361, 0x20636F, 0x206465, 0x20656C, 0x20656E, 0x206573, 0x20696E, 0x206C61, 0x206C6F, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207265, 0x207365, 7336f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20756E, 0x207920, 0x612063, 0x612064, 0x612065, 0x61206C, 0x612070, 0x616369, 0x61646F, 0x616C20, 0x617220, 0x617320, 0x6369F3, 0x636F6E, 0x646520, 0x64656C, 7346f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x646F20, 0x652064, 0x652065, 0x65206C, 0x656C20, 0x656E20, 0x656E74, 0x657320, 0x657374, 0x69656E, 0x69F36E, 0x6C6120, 0x6C6F73, 0x6E2065, 0x6E7465, 0x6F2064, 7356f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x6F2065, 0x6F6E20, 0x6F7220, 0x6F7320, 0x706172, 0x717565, 0x726120, 0x726573, 0x732064, 0x732065, 0x732070, 0x736520, 0x746520, 0x746F20, 0x756520, 0xF36E20, 7366f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 7376f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org "es" 7386f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 7396f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7406f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7416f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x206175, 0x20636F, 0x206461, 0x206465, 0x206475, 0x20656E, 0x206574, 0x206C61, 0x206C65, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207365, 0x20736F, 0x20756E, 7426f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20E020, 0x616E74, 0x617469, 0x636520, 0x636F6E, 0x646520, 0x646573, 0x647520, 0x652061, 0x652063, 0x652064, 0x652065, 0x65206C, 0x652070, 0x652073, 0x656E20, 7436f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x656E74, 0x657220, 0x657320, 0x657420, 0x657572, 0x696F6E, 0x697320, 0x697420, 0x6C6120, 0x6C6520, 0x6C6573, 0x6D656E, 0x6E2064, 0x6E6520, 0x6E7320, 0x6E7420, 7446f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x6F6E20, 0x6F6E74, 0x6F7572, 0x717565, 0x72206C, 0x726520, 0x732061, 0x732064, 0x732065, 0x73206C, 0x732070, 0x742064, 0x746520, 0x74696F, 0x756520, 0x757220, 7456f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 7466f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org "fr" 7476f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 7486f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7496f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7506f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20616C, 0x206368, 0x20636F, 0x206465, 0x206469, 0x206520, 0x20696C, 0x20696E, 0x206C61, 0x207065, 0x207072, 0x20756E, 0x612063, 0x612064, 0x612070, 0x612073, 7516f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x61746F, 0x636865, 0x636F6E, 0x64656C, 0x646920, 0x652061, 0x652063, 0x652064, 0x652069, 0x65206C, 0x652070, 0x652073, 0x656C20, 0x656C6C, 0x656E74, 0x657220, 7526f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x686520, 0x692061, 0x692063, 0x692064, 0x692073, 0x696120, 0x696C20, 0x696E20, 0x696F6E, 0x6C6120, 0x6C6520, 0x6C6920, 0x6C6C61, 0x6E6520, 0x6E6920, 0x6E6F20, 7536f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x6E7465, 0x6F2061, 0x6F2064, 0x6F2069, 0x6F2073, 0x6F6E20, 0x6F6E65, 0x706572, 0x726120, 0x726520, 0x736920, 0x746120, 0x746520, 0x746920, 0x746F20, 0x7A696F, 7546f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 7556f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org "it" 7566f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 7576f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7586f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7596f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20616C, 0x206265, 0x206461, 0x206465, 0x206469, 0x206565, 0x20656E, 0x206765, 0x206865, 0x20696E, 0x206D61, 0x206D65, 0x206F70, 0x207465, 0x207661, 0x207665, 7606f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20766F, 0x207765, 0x207A69, 0x61616E, 0x616172, 0x616E20, 0x616E64, 0x617220, 0x617420, 0x636874, 0x646520, 0x64656E, 0x646572, 0x652062, 0x652076, 0x65656E, 7616f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x656572, 0x656E20, 0x657220, 0x657273, 0x657420, 0x67656E, 0x686574, 0x696520, 0x696E20, 0x696E67, 0x697320, 0x6E2062, 0x6E2064, 0x6E2065, 0x6E2068, 0x6E206F, 7626f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x6E2076, 0x6E6465, 0x6E6720, 0x6F6E64, 0x6F6F72, 0x6F7020, 0x6F7220, 0x736368, 0x737465, 0x742064, 0x746520, 0x74656E, 0x746572, 0x76616E, 0x766572, 0x766F6F, 7636f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 7646f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org "nl" 7656f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 7666f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7676f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7686f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x206174, 0x206176, 0x206465, 0x20656E, 0x206572, 0x20666F, 0x206861, 0x206920, 0x206D65, 0x206F67, 0x2070E5, 0x207365, 0x20736B, 0x20736F, 0x207374, 0x207469, 7696f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x207669, 0x20E520, 0x616E64, 0x617220, 0x617420, 0x646520, 0x64656E, 0x646574, 0x652073, 0x656420, 0x656E20, 0x656E65, 0x657220, 0x657265, 0x657420, 0x657474, 7706f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x666F72, 0x67656E, 0x696B6B, 0x696C20, 0x696E67, 0x6B6520, 0x6B6B65, 0x6C6520, 0x6C6C65, 0x6D6564, 0x6D656E, 0x6E2073, 0x6E6520, 0x6E6720, 0x6E6765, 0x6E6E65, 7716f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x6F6720, 0x6F6D20, 0x6F7220, 0x70E520, 0x722073, 0x726520, 0x736F6D, 0x737465, 0x742073, 0x746520, 0x74656E, 0x746572, 0x74696C, 0x747420, 0x747465, 0x766572, 7726f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 7736f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org "no" 7746f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 7756f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7766f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7776f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x206120, 0x20636F, 0x206461, 0x206465, 0x20646F, 0x206520, 0x206573, 0x206D61, 0x206E6F, 0x206F20, 0x207061, 0x20706F, 0x207072, 0x207175, 0x207265, 0x207365, 7786f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20756D, 0x612061, 0x612063, 0x612064, 0x612070, 0x616465, 0x61646F, 0x616C20, 0x617220, 0x617261, 0x617320, 0x636F6D, 0x636F6E, 0x646120, 0x646520, 0x646F20, 7796f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x646F73, 0x652061, 0x652064, 0x656D20, 0x656E74, 0x657320, 0x657374, 0x696120, 0x696361, 0x6D656E, 0x6E7465, 0x6E746F, 0x6F2061, 0x6F2063, 0x6F2064, 0x6F2065, 7806f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x6F2070, 0x6F7320, 0x706172, 0x717565, 0x726120, 0x726573, 0x732061, 0x732064, 0x732065, 0x732070, 0x737461, 0x746520, 0x746F20, 0x756520, 0xE36F20, 0xE7E36F, 7816f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 7826f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org "pt" 7836f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 7846f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7856f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7866f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x206174, 0x206176, 0x206465, 0x20656E, 0x2066F6, 0x206861, 0x206920, 0x20696E, 0x206B6F, 0x206D65, 0x206F63, 0x2070E5, 0x20736B, 0x20736F, 0x207374, 0x207469, 7876f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x207661, 0x207669, 0x20E472, 0x616465, 0x616E20, 0x616E64, 0x617220, 0x617474, 0x636820, 0x646520, 0x64656E, 0x646572, 0x646574, 0x656420, 0x656E20, 0x657220, 7886f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x657420, 0x66F672, 0x67656E, 0x696C6C, 0x696E67, 0x6B6120, 0x6C6C20, 0x6D6564, 0x6E2073, 0x6E6120, 0x6E6465, 0x6E6720, 0x6E6765, 0x6E696E, 0x6F6368, 0x6F6D20, 7896f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x6F6E20, 0x70E520, 0x722061, 0x722073, 0x726120, 0x736B61, 0x736F6D, 0x742073, 0x746120, 0x746520, 0x746572, 0x74696C, 0x747420, 0x766172, 0xE47220, 0xF67220, 7906f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 7916f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org "sv" 7926f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 7936f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 7946f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 7956f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 7966f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const NGramsPlusLang ngrams_8859_2[] = { 7976f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7986f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 7996f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x206120, 0x206279, 0x20646F, 0x206A65, 0x206E61, 0x206E65, 0x206F20, 0x206F64, 0x20706F, 0x207072, 0x2070F8, 0x20726F, 0x207365, 0x20736F, 0x207374, 0x20746F, 8006f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x207620, 0x207679, 0x207A61, 0x612070, 0x636520, 0x636820, 0x652070, 0x652073, 0x652076, 0x656D20, 0x656EED, 0x686F20, 0x686F64, 0x697374, 0x6A6520, 0x6B7465, 8016f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x6C6520, 0x6C6920, 0x6E6120, 0x6EE920, 0x6EEC20, 0x6EED20, 0x6F2070, 0x6F646E, 0x6F6A69, 0x6F7374, 0x6F7520, 0x6F7661, 0x706F64, 0x706F6A, 0x70726F, 0x70F865, 8026f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x736520, 0x736F75, 0x737461, 0x737469, 0x73746E, 0x746572, 0x746EED, 0x746F20, 0x752070, 0xBE6520, 0xE16EED, 0xE9686F, 0xED2070, 0xED2073, 0xED6D20, 0xF86564, 8036f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 8046f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org "cs" 8056f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 8066f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 8076f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 8086f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x206120, 0x20617A, 0x206265, 0x206567, 0x20656C, 0x206665, 0x206861, 0x20686F, 0x206973, 0x206B65, 0x206B69, 0x206BF6, 0x206C65, 0x206D61, 0x206D65, 0x206D69, 8096f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x206E65, 0x20737A, 0x207465, 0x20E973, 0x612061, 0x61206B, 0x61206D, 0x612073, 0x616B20, 0x616E20, 0x617A20, 0x62616E, 0x62656E, 0x656779, 0x656B20, 0x656C20, 8106f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x656C65, 0x656D20, 0x656E20, 0x657265, 0x657420, 0x657465, 0x657474, 0x677920, 0x686F67, 0x696E74, 0x697320, 0x6B2061, 0x6BF67A, 0x6D6567, 0x6D696E, 0x6E2061, 8116f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x6E616B, 0x6E656B, 0x6E656D, 0x6E7420, 0x6F6779, 0x732061, 0x737A65, 0x737A74, 0x737AE1, 0x73E967, 0x742061, 0x747420, 0x74E173, 0x7A6572, 0xE16E20, 0xE97320, 8126f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 8136f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org "hu" 8146f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 8156f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 8166f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 8176f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20637A, 0x20646F, 0x206920, 0x206A65, 0x206B6F, 0x206D61, 0x206D69, 0x206E61, 0x206E69, 0x206F64, 0x20706F, 0x207072, 0x207369, 0x207720, 0x207769, 0x207779, 8186f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x207A20, 0x207A61, 0x612070, 0x612077, 0x616E69, 0x636820, 0x637A65, 0x637A79, 0x646F20, 0x647A69, 0x652070, 0x652073, 0x652077, 0x65207A, 0x65676F, 0x656A20, 8196f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x656D20, 0x656E69, 0x676F20, 0x696120, 0x696520, 0x69656A, 0x6B6120, 0x6B6920, 0x6B6965, 0x6D6965, 0x6E6120, 0x6E6961, 0x6E6965, 0x6F2070, 0x6F7761, 0x6F7769, 8206f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x706F6C, 0x707261, 0x70726F, 0x70727A, 0x727A65, 0x727A79, 0x7369EA, 0x736B69, 0x737461, 0x776965, 0x796368, 0x796D20, 0x7A6520, 0x7A6965, 0x7A7920, 0xF37720, 8216f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 8226f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org "pl" 8236f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 8246f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 8256f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org { 8266f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x206120, 0x206163, 0x206361, 0x206365, 0x20636F, 0x206375, 0x206465, 0x206469, 0x206C61, 0x206D61, 0x207065, 0x207072, 0x207365, 0x2073E3, 0x20756E, 0x20BA69, 8276f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20EE6E, 0x612063, 0x612064, 0x617265, 0x617420, 0x617465, 0x617520, 0x636172, 0x636F6E, 0x637520, 0x63E320, 0x646520, 0x652061, 0x652063, 0x652064, 0x652070, 8286f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x652073, 0x656120, 0x656920, 0x656C65, 0x656E74, 0x657374, 0x692061, 0x692063, 0x692064, 0x692070, 0x696520, 0x696920, 0x696E20, 0x6C6120, 0x6C6520, 0x6C6F72, 8296f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x6C7569, 0x6E6520, 0x6E7472, 0x6F7220, 0x70656E, 0x726520, 0x726561, 0x727520, 0x73E320, 0x746520, 0x747275, 0x74E320, 0x756920, 0x756C20, 0xBA6920, 0xEE6E20, 8306f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org }, 8316f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org "ro" 8326f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 8336f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 8346f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 8356f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const int32_t ngrams_8859_5_ru[] = { 8366f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20D220, 0x20D2DE, 0x20D4DE, 0x20D7D0, 0x20D820, 0x20DAD0, 0x20DADE, 0x20DDD0, 0x20DDD5, 0x20DED1, 0x20DFDE, 0x20DFE0, 0x20E0D0, 0x20E1DE, 0x20E1E2, 0x20E2DE, 8376f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20E7E2, 0x20EDE2, 0xD0DDD8, 0xD0E2EC, 0xD3DE20, 0xD5DBEC, 0xD5DDD8, 0xD5E1E2, 0xD5E220, 0xD820DF, 0xD8D520, 0xD8D820, 0xD8EF20, 0xDBD5DD, 0xDBD820, 0xDBECDD, 8386f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xDDD020, 0xDDD520, 0xDDD8D5, 0xDDD8EF, 0xDDDE20, 0xDDDED2, 0xDE20D2, 0xDE20DF, 0xDE20E1, 0xDED220, 0xDED2D0, 0xDED3DE, 0xDED920, 0xDEDBEC, 0xDEDC20, 0xDEE1E2, 8396f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xDFDEDB, 0xDFE0D5, 0xDFE0D8, 0xDFE0DE, 0xE0D0D2, 0xE0D5D4, 0xE1E2D0, 0xE1E2D2, 0xE1E2D8, 0xE1EF20, 0xE2D5DB, 0xE2DE20, 0xE2DEE0, 0xE2EC20, 0xE7E2DE, 0xEBE520, 8406f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 8416f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 8426f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const int32_t ngrams_8859_6_ar[] = { 8436f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20C7E4, 0x20C7E6, 0x20C8C7, 0x20D9E4, 0x20E1EA, 0x20E4E4, 0x20E5E6, 0x20E8C7, 0xC720C7, 0xC7C120, 0xC7CA20, 0xC7D120, 0xC7E420, 0xC7E4C3, 0xC7E4C7, 0xC7E4C8, 8446f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xC7E4CA, 0xC7E4CC, 0xC7E4CD, 0xC7E4CF, 0xC7E4D3, 0xC7E4D9, 0xC7E4E2, 0xC7E4E5, 0xC7E4E8, 0xC7E4EA, 0xC7E520, 0xC7E620, 0xC7E6CA, 0xC820C7, 0xC920C7, 0xC920E1, 8456f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xC920E4, 0xC920E5, 0xC920E8, 0xCA20C7, 0xCF20C7, 0xCFC920, 0xD120C7, 0xD1C920, 0xD320C7, 0xD920C7, 0xD9E4E9, 0xE1EA20, 0xE420C7, 0xE4C920, 0xE4E920, 0xE4EA20, 8466f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE520C7, 0xE5C720, 0xE5C920, 0xE5E620, 0xE620C7, 0xE720C7, 0xE7C720, 0xE8C7E4, 0xE8E620, 0xE920C7, 0xEA20C7, 0xEA20E5, 0xEA20E8, 0xEAC920, 0xEAD120, 0xEAE620, 8476f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 8486f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 8496f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const int32_t ngrams_8859_7_el[] = { 8506f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20E1ED, 0x20E1F0, 0x20E3E9, 0x20E4E9, 0x20E5F0, 0x20E720, 0x20EAE1, 0x20ECE5, 0x20EDE1, 0x20EF20, 0x20F0E1, 0x20F0EF, 0x20F0F1, 0x20F3F4, 0x20F3F5, 0x20F4E7, 8516f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20F4EF, 0xDFE120, 0xE120E1, 0xE120F4, 0xE1E920, 0xE1ED20, 0xE1F0FC, 0xE1F220, 0xE3E9E1, 0xE5E920, 0xE5F220, 0xE720F4, 0xE7ED20, 0xE7F220, 0xE920F4, 0xE9E120, 8526f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE9EADE, 0xE9F220, 0xEAE1E9, 0xEAE1F4, 0xECE520, 0xED20E1, 0xED20E5, 0xED20F0, 0xEDE120, 0xEFF220, 0xEFF520, 0xF0EFF5, 0xF0F1EF, 0xF0FC20, 0xF220E1, 0xF220E5, 8536f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xF220EA, 0xF220F0, 0xF220F4, 0xF3E520, 0xF3E720, 0xF3F4EF, 0xF4E120, 0xF4E1E9, 0xF4E7ED, 0xF4E7F2, 0xF4E9EA, 0xF4EF20, 0xF4EFF5, 0xF4F9ED, 0xF9ED20, 0xFEED20, 8546f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 8556f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 8566f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const int32_t ngrams_8859_8_I_he[] = { 8576f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20E0E5, 0x20E0E7, 0x20E0E9, 0x20E0FA, 0x20E1E9, 0x20E1EE, 0x20E4E0, 0x20E4E5, 0x20E4E9, 0x20E4EE, 0x20E4F2, 0x20E4F9, 0x20E4FA, 0x20ECE0, 0x20ECE4, 0x20EEE0, 8586f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20F2EC, 0x20F9EC, 0xE0FA20, 0xE420E0, 0xE420E1, 0xE420E4, 0xE420EC, 0xE420EE, 0xE420F9, 0xE4E5E0, 0xE5E020, 0xE5ED20, 0xE5EF20, 0xE5F820, 0xE5FA20, 0xE920E4, 8596f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE9E420, 0xE9E5FA, 0xE9E9ED, 0xE9ED20, 0xE9EF20, 0xE9F820, 0xE9FA20, 0xEC20E0, 0xEC20E4, 0xECE020, 0xECE420, 0xED20E0, 0xED20E1, 0xED20E4, 0xED20EC, 0xED20EE, 8606f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xED20F9, 0xEEE420, 0xEF20E4, 0xF0E420, 0xF0E920, 0xF0E9ED, 0xF2EC20, 0xF820E4, 0xF8E9ED, 0xF9EC20, 0xFA20E0, 0xFA20E1, 0xFA20E4, 0xFA20EC, 0xFA20EE, 0xFA20F9, 8616f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 8626f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 8636f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const int32_t ngrams_8859_8_he[] = { 8646f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20E0E5, 0x20E0EC, 0x20E4E9, 0x20E4EC, 0x20E4EE, 0x20E4F0, 0x20E9F0, 0x20ECF2, 0x20ECF9, 0x20EDE5, 0x20EDE9, 0x20EFE5, 0x20EFE9, 0x20F8E5, 0x20F8E9, 0x20FAE0, 8656f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x20FAE5, 0x20FAE9, 0xE020E4, 0xE020EC, 0xE020ED, 0xE020FA, 0xE0E420, 0xE0E5E4, 0xE0EC20, 0xE0EE20, 0xE120E4, 0xE120ED, 0xE120FA, 0xE420E4, 0xE420E9, 0xE420EC, 8666f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xE420ED, 0xE420EF, 0xE420F8, 0xE420FA, 0xE4EC20, 0xE5E020, 0xE5E420, 0xE7E020, 0xE9E020, 0xE9E120, 0xE9E420, 0xEC20E4, 0xEC20ED, 0xEC20FA, 0xECF220, 0xECF920, 8676f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0xEDE9E9, 0xEDE9F0, 0xEDE9F8, 0xEE20E4, 0xEE20ED, 0xEE20FA, 0xEEE120, 0xEEE420, 0xF2E420, 0xF920E4, 0xF920ED, 0xF920FA, 0xF9E420, 0xFAE020, 0xFAE420, 0xFAE5E9, 8686f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 8696f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 8706f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgstatic const int32_t ngrams_8859_9_tr[] = { 8716f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x206261, 0x206269, 0x206275, 0x206461, 0x206465, 0x206765, 0x206861, 0x20696C, 0x206B61, 0x206B6F, 0x206D61, 0x206F6C, 0x207361, 0x207461, 0x207665, 0x207961, 8726f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x612062, 0x616B20, 0x616C61, 0x616D61, 0x616E20, 0x616EFD, 0x617220, 0x617261, 0x6172FD, 0x6173FD, 0x617961, 0x626972, 0x646120, 0x646520, 0x646920, 0x652062, 8736f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x65206B, 0x656469, 0x656E20, 0x657220, 0x657269, 0x657369, 0x696C65, 0x696E20, 0x696E69, 0x697220, 0x6C616E, 0x6C6172, 0x6C6520, 0x6C6572, 0x6E2061, 0x6E2062, 8746f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 0x6E206B, 0x6E6461, 0x6E6465, 0x6E6520, 0x6E6920, 0x6E696E, 0x6EFD20, 0x72696E, 0x72FD6E, 0x766520, 0x796120, 0x796F72, 0xFD6E20, 0xFD6E64, 0xFD6EFD, 0xFDF0FD, 8756f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org}; 8766f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 8776f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_8859_1::~CharsetRecog_8859_1() 8786f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 8796f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 8806f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 8816f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 8826f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgUBool CharsetRecog_8859_1::match(InputText *textIn, CharsetMatch *results) const { 8836f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org const char *name = textIn->fC1Bytes? "windows-1252" : "ISO-8859-1"; 8846f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org uint32_t i; 8856f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t bestConfidenceSoFar = -1; 8866f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org for (i=0; i < ARRAY_SIZE(ngrams_8859_1) ; i++) { 8876f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org const int32_t *ngrams = ngrams_8859_1[i].ngrams; 8886f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org const char *lang = ngrams_8859_1[i].lang; 8896f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t confidence = match_sbcs(textIn, ngrams, charMap_8859_1); 8906f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if (confidence > bestConfidenceSoFar) { 8916f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org results->set(textIn, this, confidence, name, lang); 8926f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org bestConfidenceSoFar = confidence; 8936f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 8946f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 8956f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return (bestConfidenceSoFar > 0); 8966f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 8976f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 8986f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_8859_1::getName() const 8996f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 9006f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "ISO-8859-1"; 9016f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 9026f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 9036f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 9046f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_8859_2::~CharsetRecog_8859_2() 9056f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 9066f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 9076f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 9086f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 9096f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgUBool CharsetRecog_8859_2::match(InputText *textIn, CharsetMatch *results) const { 9106f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org const char *name = textIn->fC1Bytes? "windows-1250" : "ISO-8859-2"; 9116f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org uint32_t i; 9126f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t bestConfidenceSoFar = -1; 9136f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org for (i=0; i < ARRAY_SIZE(ngrams_8859_2) ; i++) { 9146f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org const int32_t *ngrams = ngrams_8859_2[i].ngrams; 9156f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org const char *lang = ngrams_8859_2[i].lang; 9166f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t confidence = match_sbcs(textIn, ngrams, charMap_8859_2); 9176f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org if (confidence > bestConfidenceSoFar) { 9186f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org results->set(textIn, this, confidence, name, lang); 9196f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org bestConfidenceSoFar = confidence; 9206f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 9216f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org } 9226f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return (bestConfidenceSoFar > 0); 9236f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 9246f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 9256f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_8859_2::getName() const 9266f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 9276f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "ISO-8859-2"; 9286f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 9296f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 9306f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 9316f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_8859_5::~CharsetRecog_8859_5() 9326f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 9336f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 9346f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 9356f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 9366f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_8859_5::getName() const 9376f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 9386f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "ISO-8859-5"; 9396f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 9406f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 9416f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_8859_5_ru::~CharsetRecog_8859_5_ru() 9426f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 9436f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 9446f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 9456f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 9466f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_8859_5_ru::getLanguage() const 9476f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 9486f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "ru"; 9496f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 9506f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 9516f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgUBool CharsetRecog_8859_5_ru::match(InputText *textIn, CharsetMatch *results) const 9526f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 9536f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t confidence = match_sbcs(textIn, ngrams_8859_5_ru, charMap_8859_5); 9546f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org results->set(textIn, this, confidence); 9556f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return (confidence > 0); 9566f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 9576f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 9586f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_8859_6::~CharsetRecog_8859_6() 9596f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 9606f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 9616f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 9626f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 9636f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_8859_6::getName() const 9646f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 9656f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "ISO-8859-6"; 9666f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 9676f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 9686f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_8859_6_ar::~CharsetRecog_8859_6_ar() 9696f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 9706f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 9716f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 9726f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 9736f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_8859_6_ar::getLanguage() const 9746f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 9756f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "ar"; 9766f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 9776f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 9786f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgUBool CharsetRecog_8859_6_ar::match(InputText *textIn, CharsetMatch *results) const 9796f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 9806f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t confidence = match_sbcs(textIn, ngrams_8859_6_ar, charMap_8859_6); 9816f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org results->set(textIn, this, confidence); 9826f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return (confidence > 0); 9836f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 9846f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 9856f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_8859_7::~CharsetRecog_8859_7() 9866f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 9876f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 9886f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 9896f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 9906f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_8859_7::getName() const 9916f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 9926f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "ISO-8859-7"; 9936f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 9946f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 9956f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_8859_7_el::~CharsetRecog_8859_7_el() 9966f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 9976f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 9986f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 9996f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 10006f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_8859_7_el::getLanguage() const 10016f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 10026f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "el"; 10036f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 10046f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 10056f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgUBool CharsetRecog_8859_7_el::match(InputText *textIn, CharsetMatch *results) const 10066f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 10076f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org const char *name = textIn->fC1Bytes? "windows-1253" : "ISO-8859-7"; 10086f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t confidence = match_sbcs(textIn, ngrams_8859_7_el, charMap_8859_7); 10096f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org results->set(textIn, this, confidence, name, "el"); 10106f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return (confidence > 0); 10116f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 10126f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 10136f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_8859_8::~CharsetRecog_8859_8() 10146f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 10156f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 10166f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 10176f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 10186f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_8859_8::getName() const 10196f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 10206f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "ISO-8859-8"; 10216f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 10226f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 10236f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_8859_8_I_he::~CharsetRecog_8859_8_I_he () 10246f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 10256f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 10266f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 10276f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 10286f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_8859_8_I_he::getName() const 10296f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 10306f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "ISO-8859-8-I"; 10316f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 10326f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 10336f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_8859_8_I_he::getLanguage() const 10346f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 10356f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "he"; 10366f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 10376f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 10386f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgUBool CharsetRecog_8859_8_I_he::match(InputText *textIn, CharsetMatch *results) const 10396f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 10406f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org const char *name = textIn->fC1Bytes? "windows-1255" : "ISO-8859-8-I"; 10416f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t confidence = match_sbcs(textIn, ngrams_8859_8_I_he, charMap_8859_8); 10426f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org results->set(textIn, this, confidence, name, "he"); 10436f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return (confidence > 0); 10446f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 10456f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 10466f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_8859_8_he::~CharsetRecog_8859_8_he() 10476f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 10486f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // od ot gnihton 10496f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 10506f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 10516f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_8859_8_he::getLanguage() const 10526f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 10536f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "he"; 10546f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 10556f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 10566f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgUBool CharsetRecog_8859_8_he::match(InputText *textIn, CharsetMatch *results) const 10576f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 10586f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org const char *name = textIn->fC1Bytes? "windows-1255" : "ISO-8859-8"; 10596f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t confidence = match_sbcs(textIn, ngrams_8859_8_he, charMap_8859_8); 10606f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org results->set(textIn, this, confidence, name, "he"); 10616f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return (confidence > 0); 10626f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 10636f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 10646f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_8859_9::~CharsetRecog_8859_9() 10656f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 10666f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 10676f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 10686f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 10696f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_8859_9::getName() const 10706f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 10716f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "ISO-8859-9"; 10726f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 10736f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 10746f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_8859_9_tr::~CharsetRecog_8859_9_tr () 10756f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 10766f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 10776f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 10786f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 10796f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_8859_9_tr::getLanguage() const 10806f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 10816f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "tr"; 10826f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 10836f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 10846f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgUBool CharsetRecog_8859_9_tr::match(InputText *textIn, CharsetMatch *results) const 10856f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 10866f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org const char *name = textIn->fC1Bytes? "windows-1254" : "ISO-8859-9"; 10876f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t confidence = match_sbcs(textIn, ngrams_8859_9_tr, charMap_8859_9); 10886f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org results->set(textIn, this, confidence, name, "tr"); 10896f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return (confidence > 0); 10906f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 10916f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 10926f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_windows_1256::~CharsetRecog_windows_1256() 10936f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 10946f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 10956f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 10966f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 10976f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_windows_1256::getName() const 10986f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 10996f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "windows-1256"; 11006f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 11016f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 11026f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_windows_1256::getLanguage() const 11036f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 11046f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "ar"; 11056f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 11066f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 11076f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgUBool CharsetRecog_windows_1256::match(InputText *textIn, CharsetMatch *results) const 11086f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 11096f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t confidence = match_sbcs(textIn, ngrams_windows_1256, charMap_windows_1256); 11106f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org results->set(textIn, this, confidence); 11116f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return (confidence > 0); 11126f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 11136f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 11146f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_windows_1251::~CharsetRecog_windows_1251() 11156f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 11166f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 11176f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 11186f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 11196f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_windows_1251::getName() const 11206f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 11216f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "windows-1251"; 11226f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 11236f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 11246f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_windows_1251::getLanguage() const 11256f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 11266f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "ru"; 11276f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 11286f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 11296f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgUBool CharsetRecog_windows_1251::match(InputText *textIn, CharsetMatch *results) const 11306f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 11316f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t confidence = match_sbcs(textIn, ngrams_windows_1251, charMap_windows_1251); 11326f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org results->set(textIn, this, confidence); 11336f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return (confidence > 0); 11346f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 11356f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 11366f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_KOI8_R::~CharsetRecog_KOI8_R() 11376f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 11386f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 11396f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 11406f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 11416f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_KOI8_R::getName() const 11426f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 11436f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "KOI8-R"; 11446f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 11456f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 11466f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_KOI8_R::getLanguage() const 11476f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 11486f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "ru"; 11496f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 11506f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 11516f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgUBool CharsetRecog_KOI8_R::match(InputText *textIn, CharsetMatch *results) const 11526f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 11536f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t confidence = match_sbcs(textIn, ngrams_KOI8_R, charMap_KOI8_R); 11546f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org results->set(textIn, this, confidence); 11556f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return (confidence > 0); 11566f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 11576f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 11586f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_IBM424_he::~CharsetRecog_IBM424_he() 11596f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 11606f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 11616f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 11626f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 11636f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_IBM424_he::getLanguage() const 11646f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 11656f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "he"; 11666f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 11676f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 11686f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_IBM424_he_rtl::~CharsetRecog_IBM424_he_rtl() 11696f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 11706f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 11716f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 11726f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 11736f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_IBM424_he_rtl::getName() const 11746f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 11756f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "IBM424_rtl"; 11766f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 11776f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 11786f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgUBool CharsetRecog_IBM424_he_rtl::match(InputText *textIn, CharsetMatch *results) const 11796f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 11806f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t confidence = match_sbcs(textIn, ngrams_IBM424_he_rtl, charMap_IBM424_he); 11816f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org results->set(textIn, this, confidence); 11826f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return (confidence > 0); 11836f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 11846f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 11856f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_IBM424_he_ltr::~CharsetRecog_IBM424_he_ltr() 11866f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 11876f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 11886f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 11896f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 11906f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_IBM424_he_ltr::getName() const 11916f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 11926f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "IBM424_ltr"; 11936f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 11946f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 11956f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgUBool CharsetRecog_IBM424_he_ltr::match(InputText *textIn, CharsetMatch *results) const 11966f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 11976f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t confidence = match_sbcs(textIn, ngrams_IBM424_he_ltr, charMap_IBM424_he); 11986f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org results->set(textIn, this, confidence); 11996f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return (confidence > 0); 12006f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 12016f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 12026f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_IBM420_ar::~CharsetRecog_IBM420_ar() 12036f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 12046f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 12056f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 12066f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 12076f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_IBM420_ar::getLanguage() const 12086f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 12096f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "ar"; 12106f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 12116f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 12126f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 12136f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgint32_t CharsetRecog_IBM420_ar::match_sbcs(InputText *det, const int32_t ngrams[], const uint8_t byteMap[]) const 12146f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 12156f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org NGramParser_IBM420 parser(ngrams, byteMap); 12166f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t result; 12176f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 12186f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org result = parser.parse(det); 12196f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 12206f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return result; 12216f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 12226f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 12236f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_IBM420_ar_rtl::~CharsetRecog_IBM420_ar_rtl() 12246f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 12256f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 12266f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 12276f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 12286f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_IBM420_ar_rtl::getName() const 12296f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 12306f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "IBM420_rtl"; 12316f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 12326f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 12336f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgUBool CharsetRecog_IBM420_ar_rtl::match(InputText *textIn, CharsetMatch *results) const 12346f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 12356f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t confidence = match_sbcs(textIn, ngrams_IBM420_ar_rtl, charMap_IBM420_ar); 12366f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org results->set(textIn, this, confidence); 12376f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return (confidence > 0); 12386f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 12396f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 12406f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgCharsetRecog_IBM420_ar_ltr::~CharsetRecog_IBM420_ar_ltr() 12416f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 12426f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org // nothing to do 12436f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 12446f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 12456f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgconst char *CharsetRecog_IBM420_ar_ltr::getName() const 12466f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 12476f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return "IBM420_ltr"; 12486f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 12496f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 12506f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgUBool CharsetRecog_IBM420_ar_ltr::match(InputText *textIn, CharsetMatch *results) const 12516f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org{ 12526f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org int32_t confidence = match_sbcs(textIn, ngrams_IBM420_ar_ltr, charMap_IBM420_ar); 12536f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org results->set(textIn, this, confidence); 12546f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org return (confidence > 0); 12556f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org} 12566f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 12576f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.orgU_NAMESPACE_END 12586f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org#endif 12596f31ac30b9092fd02a8c97e5216cf53f3e4fae4jshin@chromium.org 1260