1/*
2 **********************************************************************
3 *   Copyright (C) 2005-2013, International Business Machines
4 *   Corporation and others.  All Rights Reserved.
5 **********************************************************************
6 */
7
8#include "unicode/utypes.h"
9
10#if !UCONFIG_NO_CONVERSION
11
12#include "csrucode.h"
13#include "csmatch.h"
14
15U_NAMESPACE_BEGIN
16
17CharsetRecog_Unicode::~CharsetRecog_Unicode()
18{
19    // nothing to do
20}
21
22CharsetRecog_UTF_16_BE::~CharsetRecog_UTF_16_BE()
23{
24    // nothing to do
25}
26
27const char *CharsetRecog_UTF_16_BE::getName() const
28{
29    return "UTF-16BE";
30}
31
32// UTF-16 confidence calculation. Very simple minded, but better than nothing.
33//   Any 8 bit non-control characters bump the confidence up. These have a zero high byte,
34//     and are very likely to be UTF-16, although they could also be part of a UTF-32 code.
35//   NULs are a contra-indication, they will appear commonly if the actual encoding is UTF-32.
36//   NULs should be rare in actual text.
37
38static int32_t adjustConfidence(UChar codeUnit, int32_t confidence) {
39    if (codeUnit == 0) {
40        confidence -= 10;
41    } else if ((codeUnit >= 0x20 && codeUnit <= 0xff) || codeUnit == 0x0a) {
42        confidence += 10;
43    }
44    if (confidence < 0) {
45        confidence = 0;
46    } else if (confidence > 100) {
47        confidence = 100;
48    }
49    return confidence;
50}
51
52
53UBool CharsetRecog_UTF_16_BE::match(InputText* textIn, CharsetMatch *results) const
54{
55    const uint8_t *input = textIn->fRawInput;
56    int32_t confidence = 10;
57    int32_t length = textIn->fRawLength;
58
59    int32_t bytesToCheck = (length > 30) ? 30 : length;
60    for (int32_t charIndex=0; charIndex<bytesToCheck-1; charIndex+=2) {
61        UChar codeUnit = (input[charIndex] << 8) | input[charIndex + 1];
62        if (charIndex == 0 && codeUnit == 0xFEFF) {
63            confidence = 100;
64            break;
65        }
66        confidence = adjustConfidence(codeUnit, confidence);
67        if (confidence == 0 || confidence == 100) {
68            break;
69        }
70    }
71    if (bytesToCheck < 4 && confidence < 100) {
72        confidence = 0;
73    }
74    results->set(textIn, this, confidence);
75    return (confidence > 0);
76}
77
78CharsetRecog_UTF_16_LE::~CharsetRecog_UTF_16_LE()
79{
80    // nothing to do
81}
82
83const char *CharsetRecog_UTF_16_LE::getName() const
84{
85    return "UTF-16LE";
86}
87
88UBool CharsetRecog_UTF_16_LE::match(InputText* textIn, CharsetMatch *results) const
89{
90    const uint8_t *input = textIn->fRawInput;
91    int32_t confidence = 10;
92    int32_t length = textIn->fRawLength;
93
94    int32_t bytesToCheck = (length > 30) ? 30 : length;
95    for (int32_t charIndex=0; charIndex<bytesToCheck-1; charIndex+=2) {
96        UChar codeUnit = input[charIndex] | (input[charIndex + 1] << 8);
97        if (charIndex == 0 && codeUnit == 0xFEFF) {
98            confidence = 100;     // UTF-16 BOM
99            if (length >= 4 && input[2] == 0 && input[3] == 0) {
100                confidence = 0;   // UTF-32 BOM
101            }
102            break;
103        }
104        confidence = adjustConfidence(codeUnit, confidence);
105        if (confidence == 0 || confidence == 100) {
106            break;
107        }
108    }
109    if (bytesToCheck < 4 && confidence < 100) {
110        confidence = 0;
111    }
112    results->set(textIn, this, confidence);
113    return (confidence > 0);
114}
115
116CharsetRecog_UTF_32::~CharsetRecog_UTF_32()
117{
118    // nothing to do
119}
120
121UBool CharsetRecog_UTF_32::match(InputText* textIn, CharsetMatch *results) const
122{
123    const uint8_t *input = textIn->fRawInput;
124    int32_t limit = (textIn->fRawLength / 4) * 4;
125    int32_t numValid = 0;
126    int32_t numInvalid = 0;
127    bool hasBOM = FALSE;
128    int32_t confidence = 0;
129
130    if (limit > 0 && getChar(input, 0) == 0x0000FEFFUL) {
131        hasBOM = TRUE;
132    }
133
134    for(int32_t i = 0; i < limit; i += 4) {
135        int32_t ch = getChar(input, i);
136
137        if (ch < 0 || ch >= 0x10FFFF || (ch >= 0xD800 && ch <= 0xDFFF)) {
138            numInvalid += 1;
139        } else {
140            numValid += 1;
141        }
142    }
143
144
145    // Cook up some sort of confidence score, based on presense of a BOM
146    //    and the existence of valid and/or invalid multi-byte sequences.
147    if (hasBOM && numInvalid==0) {
148        confidence = 100;
149    } else if (hasBOM && numValid > numInvalid*10) {
150        confidence = 80;
151    } else if (numValid > 3 && numInvalid == 0) {
152        confidence = 100;
153    } else if (numValid > 0 && numInvalid == 0) {
154        confidence = 80;
155    } else if (numValid > numInvalid*10) {
156        // Probably corruput UTF-32BE data.  Valid sequences aren't likely by chance.
157        confidence = 25;
158    }
159
160    results->set(textIn, this, confidence);
161    return (confidence > 0);
162}
163
164CharsetRecog_UTF_32_BE::~CharsetRecog_UTF_32_BE()
165{
166    // nothing to do
167}
168
169const char *CharsetRecog_UTF_32_BE::getName() const
170{
171    return "UTF-32BE";
172}
173
174int32_t CharsetRecog_UTF_32_BE::getChar(const uint8_t *input, int32_t index) const
175{
176    return input[index + 0] << 24 | input[index + 1] << 16 |
177           input[index + 2] <<  8 | input[index + 3];
178}
179
180CharsetRecog_UTF_32_LE::~CharsetRecog_UTF_32_LE()
181{
182    // nothing to do
183}
184
185const char *CharsetRecog_UTF_32_LE::getName() const
186{
187    return "UTF-32LE";
188}
189
190int32_t CharsetRecog_UTF_32_LE::getChar(const uint8_t *input, int32_t index) const
191{
192    return input[index + 3] << 24 | input[index + 2] << 16 |
193           input[index + 1] <<  8 | input[index + 0];
194}
195
196U_NAMESPACE_END
197#endif
198
199