1ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru/* 2ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru****************************************************************************** 3ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru* 48393335b955da7340c9f19b1b4b2d6c0c2c04be7Craig Cornelius* Copyright (C) 2003-2013, International Business Machines 5ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru* Corporation and others. All Rights Reserved. 6ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru* 7ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru****************************************************************************** 8ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru* file name: ucnv_ext.h 9ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru* encoding: US-ASCII 10ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru* tab size: 8 (not used) 11ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru* indentation:4 12ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru* 13ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru* created on: 2003jun13 14ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru* created by: Markus W. Scherer 15ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru* 16ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru* Conversion extensions 17ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru*/ 18ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 19ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#ifndef __UCNV_EXT_H__ 20ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define __UCNV_EXT_H__ 21ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 22ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#include "unicode/utypes.h" 23ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 24ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#if !UCONFIG_NO_CONVERSION 25ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 26ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#include "unicode/ucnv.h" 27ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#include "ucnv_cnv.h" 28ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 29ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru/* 30ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * See icuhtml/design/conversion/conversion_extensions.html 31ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 328393335b955da7340c9f19b1b4b2d6c0c2c04be7Craig Cornelius * Conversion extensions serve three purposes: 33ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 1. They support m:n mappings. 34ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 2. They support extension-only conversion files that are used together 35ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * with the regular conversion data in base files. 368393335b955da7340c9f19b1b4b2d6c0c2c04be7Craig Cornelius * 3. They support mappings with more complicated meta data, 378393335b955da7340c9f19b1b4b2d6c0c2c04be7Craig Cornelius * for example "good one-way" mappings (|4). 38ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 39ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * A base file may contain an extension table (explicitly requested or 40ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * implicitly generated for m:n mappings), but its extension table is not 41ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * used when an extension-only file is used. 42ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 43ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * It is an error if a base file contains any regular (not extension) mapping 44ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * from the same sequence as a mapping in the extension file 45ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * because the base mapping would hide the extension mapping. 46ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 47ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 48ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Data for conversion extensions: 49ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 50ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * One set of data structures per conversion direction (to/from Unicode). 51ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * The data structures are sorted by input units to allow for binary search. 52ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Input sequences of more than one unit are handled like contraction tables 53ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * in collation: 54ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * The lookup value of a unit points to another table that is to be searched 55ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * for the next unit, recursively. 56ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 57ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * For conversion from Unicode, the initial code point is looked up in 58ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * a 3-stage trie for speed, 59ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * with an additional table of unique results to save space. 60ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 61ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Long output strings are stored in separate arrays, with length and index 62ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * in the lookup tables. 63ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Output results also include a flag distinguishing roundtrip from 64ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * (reverse) fallback mappings. 65ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 66ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Input Unicode strings must not begin or end with unpaired surrogates 67ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * to avoid problems with matches on parts of surrogate pairs. 68ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 69ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Mappings from multiple characters (code points or codepage state 70ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * table sequences) must be searched preferring the longest match. 71ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * For this to work and be efficient, the variable-width table must contain 72ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * all mappings that contain prefixes of the multiple characters. 73ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * If an extension table is built on top of a base table in another file 74ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * and a base table entry is a prefix of a multi-character mapping, then 75ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * this is an error. 76ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 77ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 78ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Implementation note: 79ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 80ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Currently, the parser and several checks in the code limit the number 81ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * of UChars or bytes in a mapping to 82ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * UCNV_EXT_MAX_UCHARS and UCNV_EXT_MAX_BYTES, respectively, 83ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * which are output value limits in the data structure. 84ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 85ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * For input, this is not strictly necessary - it is a hard limit only for the 86ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * buffers in UConverter that are used to store partial matches. 87ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 88ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Input sequences could otherwise be arbitrarily long if partial matches 89ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * need not be stored (i.e., if a sequence does not span several buffers with too 90ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * many units before the last buffer), although then results would differ 91ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * depending on whether partial matches exceed the limits or not, 92ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * which depends on the pattern of buffer sizes. 93ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 94ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 95ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Data structure: 96ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 97ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * int32_t indexes[>=32]; 98ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 99ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Array of indexes and lengths etc. The length of the array is at least 32. 100ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * The actual length is stored in indexes[0] to be forward compatible. 101ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 102ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Each index to another array is the number of bytes from indexes[]. 103ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Each length of an array is the number of array base units in that array. 104ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 105ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Some of the structures may not be present, in which case their indexes 106ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * and lengths are 0. 107ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 108ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Usage of indexes[i]: 109ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [0] length of indexes[] 110ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 111ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * // to Unicode table 112ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [1] index of toUTable[] (array of uint32_t) 113ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [2] length of toUTable[] 114ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [3] index of toUUChars[] (array of UChar) 115ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [4] length of toUUChars[] 116ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 117ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * // from Unicode table, not for the initial code point 118ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [5] index of fromUTableUChars[] (array of UChar) 119ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [6] index of fromUTableValues[] (array of uint32_t) 120ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [7] length of fromUTableUChars[] and fromUTableValues[] 121ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [8] index of fromUBytes[] (array of char) 122ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [9] length of fromUBytes[] 123ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 124ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * // from Unicode trie for initial-code point lookup 125ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [10] index of fromUStage12[] (combined array of uint16_t for stages 1 & 2) 126ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [11] length of stage 1 portion of fromUStage12[] 127ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [12] length of fromUStage12[] 128ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [13] index of fromUStage3[] (array of uint16_t indexes into fromUStage3b[]) 129ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [14] length of fromUStage3[] 130ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [15] index of fromUStage3b[] (array of uint32_t like fromUTableValues[]) 131ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [16] length of fromUStage3b[] 132ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 133ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [17] Bit field containing numbers of bytes: 134ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 31..24 reserved, 0 135ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 23..16 maximum input bytes 136ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 15.. 8 maximum output bytes 137ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 7.. 0 maximum bytes per UChar 138ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 139ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [18] Bit field containing numbers of UChars: 140ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 31..24 reserved, 0 141ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 23..16 maximum input UChars 142ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 15.. 8 maximum output UChars 143ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 7.. 0 maximum UChars per byte 144ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 145ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [19] Bit field containing flags: 146ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * (extension table unicodeMask) 147ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 1 UCNV_HAS_SURROGATES flag for the extension table 148ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 0 UCNV_HAS_SUPPLEMENTARY flag for the extension table 149ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 150ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [20]..[30] reserved, 0 151ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [31] number of bytes for the entire extension structure 152ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * [>31] reserved; there are indexes[0] indexes 153ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 154ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 155ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * uint32_t toUTable[]; 156ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 157ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Array of byte/value pairs for lookups for toUnicode conversion. 158ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * The array is partitioned into sections like collation contraction tables. 159ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Each section contains one word with the number of following words and 160ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * a default value for when the lookup in this section yields no match. 161ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 162ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * A section is sorted in ascending order of input bytes, 163ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * allowing for fast linear or binary searches. 164ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * The builder may store entries for a contiguous range of byte values 165ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * (compare difference between the first and last one with count), 166ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * which then allows for direct array access. 167ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * The builder should always do this for the initial table section. 168ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 169ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Entries may have 0 values, see below. 170ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * No two entries in a section have the same byte values. 171ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 172ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Each uint32_t contains an input byte value in bits 31..24 and the 173ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * corresponding lookup value in bits 23..0. 174ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Interpret the value as follows: 175ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * if(value==0) { 176ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * no match, see below 177ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * } else if(value<0x1f0000) { 178ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * partial match - use value as index to the next toUTable section 179ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * and match the next unit; (value indexes toUTable[value]) 180ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * } else { 181ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * if(bit 23 set) { 182ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * roundtrip; 183ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * } else { 184ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * fallback; 185ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * } 186ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * unset value bit 23; 187ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * if(value<=0x2fffff) { 188ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * (value-0x1f0000) is a code point; (BMP: value<=0x1fffff) 189ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * } else { 190ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * bits 17..0 (value&0x3ffff) is an index to 191ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * the result UChars in toUUChars[]; (0 indexes toUUChars[0]) 192ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * length of the result=((value>>18)-12); (length=0..19) 193ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * } 194ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * } 195ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 196ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * The first word in a section contains the number of following words in the 197ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * input byte position (bits 31..24, number=1..0xff). 198ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * The value of the initial word is used when the current byte is not found 199ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * in this section. 200ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * If the value is not 0, then it represents a result as above. 201ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * If the value is 0, then the search has to return a shorter match with an 202ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * earlier default value as the result, or result in "unmappable" even for the 203ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * initial bytes. 204ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * If the value is 0 for the initial toUTable entry, then the initial byte 205ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * does not start any mapping input. 206ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 207ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 208ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * UChar toUUChars[]; 209ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 210ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Contains toUnicode mapping results, stored as sequences of UChars. 211ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Indexes and lengths stored in the toUTable[]. 212ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 213ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 214ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * UChar fromUTableUChars[]; 215ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * uint32_t fromUTableValues[]; 216ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 217ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * The fromUTable is split into two arrays, but works otherwise much like 218ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * the toUTable. The array is partitioned into sections like collation 219ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * contraction tables and toUTable. 220ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * A row in the table consists of same-index entries in fromUTableUChars[] 221ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * and fromUTableValues[]. 222ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 223ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Interpret a value as follows: 224ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * if(value==0) { 225ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * no match, see below 226ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * } else if(value<=0xffffff) { (bits 31..24 are 0) 227ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * partial match - use value as index to the next fromUTable section 228ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * and match the next unit; (value indexes fromUTable[value]) 229ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * } else { 230ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * if(value==0x80000001) { 231ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * return no mapping, but request for <subchar1>; 232ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * } 233ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * if(bit 31 set) { 2348393335b955da7340c9f19b1b4b2d6c0c2c04be7Craig Cornelius * roundtrip (|0); 2358393335b955da7340c9f19b1b4b2d6c0c2c04be7Craig Cornelius * } else if(bit 30 set) { 2368393335b955da7340c9f19b1b4b2d6c0c2c04be7Craig Cornelius * "good one-way" mapping (|4); -- new in ICU4C 51, _MBCSHeader.version 5.4/4.4 237ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * } else { 2388393335b955da7340c9f19b1b4b2d6c0c2c04be7Craig Cornelius * normal fallback (|1); 239ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * } 2408393335b955da7340c9f19b1b4b2d6c0c2c04be7Craig Cornelius * // bit 29 reserved, 0 241ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * length=(value>>24)&0x1f; (bits 28..24) 242ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * if(length==1..3) { 243ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * bits 23..0 contain 1..3 bytes, padded with 00s on the left; 244ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * } else { 245ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * bits 23..0 (value&0xffffff) is an index to 246ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * the result bytes in fromUBytes[]; (0 indexes fromUBytes[0]) 247ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * } 248ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * } 249ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 250ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * The first pair in a section contains the number of following pairs in the 251ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * UChar position (16 bits, number=1..0xffff). 252ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * The value of the initial pair is used when the current UChar is not found 253ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * in this section. 254ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * If the value is not 0, then it represents a result as above. 255ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * If the value is 0, then the search has to return a shorter match with an 256ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * earlier default value as the result, or result in "unmappable" even for the 257ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * initial UChars. 258ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 259ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * If the from Unicode trie is present, then the from Unicode search tables 260ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * are not used for initial code points. 261ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * In this case, the first entries (index 0) in the tables are not used 262ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * (reserved, set to 0) because a value of 0 is used in trie results 263ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * to indicate no mapping. 264ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 265ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 266ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * uint16_t fromUStage12[]; 267ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 268ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Stages 1 & 2 of a trie that maps an initial code point. 269ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Indexes in stage 1 are all offset by the length of stage 1 so that the 270ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * same array pointer can be used for both stages. 271ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * If (c>>10)>=(length of stage 1) then c does not start any mapping. 272ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Same bit distribution as for regular conversion tries. 273ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 274ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 275ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * uint16_t fromUStage3[]; 276ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * uint32_t fromUStage3b[]; 277ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 278ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Stage 3 of the trie. The first array simply contains indexes to the second, 279ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * which contains words in the same format as fromUTableValues[]. 280ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Use a stage 3 granularity of 4, which allows for 256k stage 3 entries, 281ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * and 16-bit entries in stage 3 allow for 64k stage 3b entries. 282ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * The stage 3 granularity means that the stage 2 entry needs to be left-shifted. 283ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 284ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Two arrays are used because it is expected that more than half of the stage 3 285ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * entries will be zero. The 16-bit index stage 3 array saves space even 286ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * considering storing a total of 6 bytes per non-zero entry in both arrays 287ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * together. 288ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Using a stage 3 granularity of >1 diminishes the compactability in that stage 289ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * but provides a larger effective addressing space in stage 2. 290ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * All but the final result stage use 16-bit entries to save space. 291ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 292ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * fromUStage3b[] contains a zero for "no mapping" at its index 0, 293ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * and may contain UCNV_EXT_FROM_U_SUBCHAR1 at index 1 for "<subchar1> SUB mapping" 294ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * (i.e., "no mapping" with preference for <subchar1> rather than <subchar>), 295ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * and all other items are unique non-zero results. 296ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 297ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * The default value of a fromUTableValues[] section that is referenced 298ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * _directly_ from a fromUStage3b[] item may also be UCNV_EXT_FROM_U_SUBCHAR1, 299ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * but this value must not occur anywhere else in fromUTableValues[] 300ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * because "no mapping" is always a property of a single code point, 301ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * never of multiple. 302ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 303ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 304ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * char fromUBytes[]; 305ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * 306ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Contains fromUnicode mapping results, stored as sequences of chars. 307ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru * Indexes and lengths stored in the fromUTableValues[]. 308ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru */ 309ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queruenum { 310ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_INDEXES_LENGTH, /* 0 */ 311ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 312ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_TO_U_INDEX, /* 1 */ 313ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_TO_U_LENGTH, 314ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_TO_U_UCHARS_INDEX, 315ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_TO_U_UCHARS_LENGTH, 316ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 317ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_FROM_U_UCHARS_INDEX, /* 5 */ 318ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_FROM_U_VALUES_INDEX, 319ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_FROM_U_LENGTH, 320ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_FROM_U_BYTES_INDEX, 321ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_FROM_U_BYTES_LENGTH, 322ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 323ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_FROM_U_STAGE_12_INDEX, /* 10 */ 324ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_FROM_U_STAGE_1_LENGTH, 325ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_FROM_U_STAGE_12_LENGTH, 326ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_FROM_U_STAGE_3_INDEX, 327ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_FROM_U_STAGE_3_LENGTH, 328ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_FROM_U_STAGE_3B_INDEX, 329ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_FROM_U_STAGE_3B_LENGTH, 330ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 331ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_COUNT_BYTES, /* 17 */ 332ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_COUNT_UCHARS, 333ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_FLAGS, 334ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 335ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_RESERVED_INDEX, /* 20, moves with additional indexes */ 336ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 337ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_SIZE=31, 338ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UCNV_EXT_INDEXES_MIN_LENGTH=32 339ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru}; 340ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 341ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru/* get the pointer to an extension array from indexes[index] */ 342ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_ARRAY(indexes, index, itemType) \ 343ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru ((const itemType *)((const char *)(indexes)+(indexes)[index])) 344ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 345ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_GET_MAX_BYTES_PER_UCHAR(indexes) \ 346ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru ((indexes)[UCNV_EXT_COUNT_BYTES]&0xff) 347ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 348ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru/* internal API ------------------------------------------------------------- */ 349ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 350ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste QueruU_CFUNC UBool 351ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queruucnv_extInitialMatchToU(UConverter *cnv, const int32_t *cx, 352ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru int32_t firstLength, 353ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru const char **src, const char *srcLimit, 354ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UChar **target, const UChar *targetLimit, 355ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru int32_t **offsets, int32_t srcIndex, 356ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UBool flush, 357ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UErrorCode *pErrorCode); 358ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 359ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste QueruU_CFUNC UChar32 360ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queruucnv_extSimpleMatchToU(const int32_t *cx, 361ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru const char *source, int32_t length, 362ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UBool useFallback); 363ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 364ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste QueruU_CFUNC void 365ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queruucnv_extContinueMatchToU(UConverter *cnv, 366ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UConverterToUnicodeArgs *pArgs, int32_t srcIndex, 367ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UErrorCode *pErrorCode); 368ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 369ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 370ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste QueruU_CFUNC UBool 371ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queruucnv_extInitialMatchFromU(UConverter *cnv, const int32_t *cx, 372ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UChar32 cp, 373ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru const UChar **src, const UChar *srcLimit, 374ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru char **target, const char *targetLimit, 375ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru int32_t **offsets, int32_t srcIndex, 376ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UBool flush, 377ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UErrorCode *pErrorCode); 378ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 379ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste QueruU_CFUNC int32_t 380ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queruucnv_extSimpleMatchFromU(const int32_t *cx, 381ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UChar32 cp, uint32_t *pValue, 382ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UBool useFallback); 383ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 384ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste QueruU_CFUNC void 385ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queruucnv_extContinueMatchFromU(UConverter *cnv, 386ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UConverterFromUnicodeArgs *pArgs, int32_t srcIndex, 387ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UErrorCode *pErrorCode); 388ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 389c0f3e2506e4cc62ff8c220fe72849728e9d6cecfJean-Baptiste Queru/* 390c0f3e2506e4cc62ff8c220fe72849728e9d6cecfJean-Baptiste Queru * Add code points and strings to the set according to the extension mappings. 391c0f3e2506e4cc62ff8c220fe72849728e9d6cecfJean-Baptiste Queru * Limitation on the UConverterSetFilter: 392c0f3e2506e4cc62ff8c220fe72849728e9d6cecfJean-Baptiste Queru * The filters currently assume that they are used with 1:1 mappings. 393c0f3e2506e4cc62ff8c220fe72849728e9d6cecfJean-Baptiste Queru * They only apply to single input code points, and then they pass through 394c0f3e2506e4cc62ff8c220fe72849728e9d6cecfJean-Baptiste Queru * only mappings with single-charset-code results. 395c0f3e2506e4cc62ff8c220fe72849728e9d6cecfJean-Baptiste Queru * For example, the Shift-JIS filter only works for 2-byte results and tests 396c0f3e2506e4cc62ff8c220fe72849728e9d6cecfJean-Baptiste Queru * that those 2 bytes are in the JIS X 0208 range of Shift-JIS. 397c0f3e2506e4cc62ff8c220fe72849728e9d6cecfJean-Baptiste Queru */ 398ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste QueruU_CFUNC void 399ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queruucnv_extGetUnicodeSet(const UConverterSharedData *sharedData, 400ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru const USetAdder *sa, 401ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UConverterUnicodeSet which, 402c0f3e2506e4cc62ff8c220fe72849728e9d6cecfJean-Baptiste Queru UConverterSetFilter filter, 403ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru UErrorCode *pErrorCode); 404ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 405ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru/* toUnicode helpers -------------------------------------------------------- */ 406ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 407ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_TO_U_BYTE_SHIFT 24 408ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_TO_U_VALUE_MASK 0xffffff 409ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_TO_U_MIN_CODE_POINT 0x1f0000 410ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_TO_U_MAX_CODE_POINT 0x2fffff 411ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_TO_U_ROUNDTRIP_FLAG ((uint32_t)1<<23) 412ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_TO_U_INDEX_MASK 0x3ffff 413ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_TO_U_LENGTH_SHIFT 18 414ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_TO_U_LENGTH_OFFSET 12 415ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 416ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru/* maximum number of indexed UChars */ 417ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_MAX_UCHARS 19 418ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 419ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_TO_U_MAKE_WORD(byte, value) (((uint32_t)(byte)<<UCNV_EXT_TO_U_BYTE_SHIFT)|(value)) 420ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 421ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_TO_U_GET_BYTE(word) ((word)>>UCNV_EXT_TO_U_BYTE_SHIFT) 422ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_TO_U_GET_VALUE(word) ((word)&UCNV_EXT_TO_U_VALUE_MASK) 423ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 424ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_TO_U_IS_PARTIAL(value) ((value)<UCNV_EXT_TO_U_MIN_CODE_POINT) 425ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_TO_U_GET_PARTIAL_INDEX(value) (value) 426ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 427ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_TO_U_IS_ROUNDTRIP(value) (((value)&UCNV_EXT_TO_U_ROUNDTRIP_FLAG)!=0) 428ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_TO_U_MASK_ROUNDTRIP(value) ((value)&~UCNV_EXT_TO_U_ROUNDTRIP_FLAG) 429ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 430ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru/* use after masking off the roundtrip flag */ 431ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_TO_U_IS_CODE_POINT(value) ((value)<=UCNV_EXT_TO_U_MAX_CODE_POINT) 432ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_TO_U_GET_CODE_POINT(value) ((value)-UCNV_EXT_TO_U_MIN_CODE_POINT) 433ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 434ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_TO_U_GET_INDEX(value) ((value)&UCNV_EXT_TO_U_INDEX_MASK) 435ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_TO_U_GET_LENGTH(value) (((value)>>UCNV_EXT_TO_U_LENGTH_SHIFT)-UCNV_EXT_TO_U_LENGTH_OFFSET) 436ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 437ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru/* fromUnicode helpers ------------------------------------------------------ */ 438ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 439ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru/* most trie constants are shared with ucnvmbcs.h */ 440ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 441ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru/* see similar utrie.h UTRIE_INDEX_SHIFT and UTRIE_DATA_GRANULARITY */ 442ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_STAGE_2_LEFT_SHIFT 2 443ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_STAGE_3_GRANULARITY 4 444ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 445ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru/* trie access, returns the stage 3 value=index to stage 3b; s1Index=c>>10 */ 446ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_FROM_U(stage12, stage3, s1Index, c) \ 447ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru (stage3)[ ((int32_t)(stage12)[ (stage12)[s1Index] +(((c)>>4)&0x3f) ]<<UCNV_EXT_STAGE_2_LEFT_SHIFT) +((c)&0xf) ] 448ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 449ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_FROM_U_LENGTH_SHIFT 24 450ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_FROM_U_ROUNDTRIP_FLAG ((uint32_t)1<<31) 4518393335b955da7340c9f19b1b4b2d6c0c2c04be7Craig Cornelius#define UCNV_EXT_FROM_U_GOOD_ONE_WAY_FLAG 0x40000000 4528393335b955da7340c9f19b1b4b2d6c0c2c04be7Craig Cornelius#define UCNV_EXT_FROM_U_STATUS_MASK 0xc0000000 4538393335b955da7340c9f19b1b4b2d6c0c2c04be7Craig Cornelius#define UCNV_EXT_FROM_U_RESERVED_MASK 0x20000000 454ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_FROM_U_DATA_MASK 0xffffff 455ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 456ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru/* special value for "no mapping" to <subchar1> (impossible roundtrip to 0 bytes, value 01) */ 457ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_FROM_U_SUBCHAR1 0x80000001 458ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 459ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru/* at most 3 bytes in the lower part of the value */ 460ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_FROM_U_MAX_DIRECT_LENGTH 3 461ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 462ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru/* maximum number of indexed bytes */ 463ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_MAX_BYTES 0x1f 464ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 465ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_FROM_U_IS_PARTIAL(value) (((value)>>UCNV_EXT_FROM_U_LENGTH_SHIFT)==0) 466ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_FROM_U_GET_PARTIAL_INDEX(value) (value) 467ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 468ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_FROM_U_IS_ROUNDTRIP(value) (((value)&UCNV_EXT_FROM_U_ROUNDTRIP_FLAG)!=0) 469ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_FROM_U_MASK_ROUNDTRIP(value) ((value)&~UCNV_EXT_FROM_U_ROUNDTRIP_FLAG) 470ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 471ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru/* get length; masks away all other bits */ 472ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_FROM_U_GET_LENGTH(value) (int32_t)(((value)>>UCNV_EXT_FROM_U_LENGTH_SHIFT)&UCNV_EXT_MAX_BYTES) 473ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 474ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru/* get bytes or bytes index */ 475ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#define UCNV_EXT_FROM_U_GET_DATA(value) ((value)&UCNV_EXT_FROM_U_DATA_MASK) 476ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 477ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#endif 478ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru 479ac04d0bbe12b3ef54518635711412f178cb4d16Jean-Baptiste Queru#endif 480