1/*
2 * Copyright (C) 2004, 2006, 2008 Apple Inc. All rights reserved.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 *    notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 *    notice, this list of conditions and the following disclaimer in the
11 *    documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
14 * EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
16 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE COMPUTER, INC. OR
17 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
18 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
19 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
20 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
21 * OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
23 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25
26#include "config.h"
27#include "wtf/text/TextCodecLatin1.h"
28
29#include "wtf/text/TextCodecASCIIFastPath.h"
30#include "wtf/PassOwnPtr.h"
31#include "wtf/text/CString.h"
32#include "wtf/text/StringBuffer.h"
33#include "wtf/text/WTFString.h"
34
35using namespace WTF;
36
37namespace WTF {
38
39static const UChar table[256] = {
40    0x0000, 0x0001, 0x0002, 0x0003, 0x0004, 0x0005, 0x0006, 0x0007, // 00-07
41    0x0008, 0x0009, 0x000A, 0x000B, 0x000C, 0x000D, 0x000E, 0x000F, // 08-0F
42    0x0010, 0x0011, 0x0012, 0x0013, 0x0014, 0x0015, 0x0016, 0x0017, // 10-17
43    0x0018, 0x0019, 0x001A, 0x001B, 0x001C, 0x001D, 0x001E, 0x001F, // 18-1F
44    0x0020, 0x0021, 0x0022, 0x0023, 0x0024, 0x0025, 0x0026, 0x0027, // 20-27
45    0x0028, 0x0029, 0x002A, 0x002B, 0x002C, 0x002D, 0x002E, 0x002F, // 28-2F
46    0x0030, 0x0031, 0x0032, 0x0033, 0x0034, 0x0035, 0x0036, 0x0037, // 30-37
47    0x0038, 0x0039, 0x003A, 0x003B, 0x003C, 0x003D, 0x003E, 0x003F, // 38-3F
48    0x0040, 0x0041, 0x0042, 0x0043, 0x0044, 0x0045, 0x0046, 0x0047, // 40-47
49    0x0048, 0x0049, 0x004A, 0x004B, 0x004C, 0x004D, 0x004E, 0x004F, // 48-4F
50    0x0050, 0x0051, 0x0052, 0x0053, 0x0054, 0x0055, 0x0056, 0x0057, // 50-57
51    0x0058, 0x0059, 0x005A, 0x005B, 0x005C, 0x005D, 0x005E, 0x005F, // 58-5F
52    0x0060, 0x0061, 0x0062, 0x0063, 0x0064, 0x0065, 0x0066, 0x0067, // 60-67
53    0x0068, 0x0069, 0x006A, 0x006B, 0x006C, 0x006D, 0x006E, 0x006F, // 68-6F
54    0x0070, 0x0071, 0x0072, 0x0073, 0x0074, 0x0075, 0x0076, 0x0077, // 70-77
55    0x0078, 0x0079, 0x007A, 0x007B, 0x007C, 0x007D, 0x007E, 0x007F, // 78-7F
56    0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, // 80-87
57    0x02C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F, // 88-8F
58    0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, // 90-97
59    0x02DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178, // 98-9F
60    0x00A0, 0x00A1, 0x00A2, 0x00A3, 0x00A4, 0x00A5, 0x00A6, 0x00A7, // A0-A7
61    0x00A8, 0x00A9, 0x00AA, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x00AF, // A8-AF
62    0x00B0, 0x00B1, 0x00B2, 0x00B3, 0x00B4, 0x00B5, 0x00B6, 0x00B7, // B0-B7
63    0x00B8, 0x00B9, 0x00BA, 0x00BB, 0x00BC, 0x00BD, 0x00BE, 0x00BF, // B8-BF
64    0x00C0, 0x00C1, 0x00C2, 0x00C3, 0x00C4, 0x00C5, 0x00C6, 0x00C7, // C0-C7
65    0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF, // C8-CF
66    0x00D0, 0x00D1, 0x00D2, 0x00D3, 0x00D4, 0x00D5, 0x00D6, 0x00D7, // D0-D7
67    0x00D8, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x00DD, 0x00DE, 0x00DF, // D8-DF
68    0x00E0, 0x00E1, 0x00E2, 0x00E3, 0x00E4, 0x00E5, 0x00E6, 0x00E7, // E0-E7
69    0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF, // E8-EF
70    0x00F0, 0x00F1, 0x00F2, 0x00F3, 0x00F4, 0x00F5, 0x00F6, 0x00F7, // F0-F7
71    0x00F8, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x00FD, 0x00FE, 0x00FF  // F8-FF
72};
73
74void TextCodecLatin1::registerEncodingNames(EncodingNameRegistrar registrar)
75{
76    registrar("windows-1252", "windows-1252");
77    registrar("ISO-8859-1", "ISO-8859-1");
78    registrar("US-ASCII", "US-ASCII");
79
80    registrar("WinLatin1", "windows-1252");
81    registrar("ibm-1252", "windows-1252");
82    registrar("ibm-1252_P100-2000", "windows-1252");
83
84    registrar("CP819", "ISO-8859-1");
85    registrar("IBM819", "ISO-8859-1");
86    registrar("csISOLatin1", "ISO-8859-1");
87    registrar("iso-ir-100", "ISO-8859-1");
88    registrar("iso_8859-1:1987", "ISO-8859-1");
89    registrar("l1", "ISO-8859-1");
90    registrar("latin1", "ISO-8859-1");
91
92    registrar("ANSI_X3.4-1968", "US-ASCII");
93    registrar("ANSI_X3.4-1986", "US-ASCII");
94    registrar("ASCII", "US-ASCII");
95    registrar("IBM367", "US-ASCII");
96    registrar("ISO646-US", "US-ASCII");
97    registrar("ISO_646.irv:1991", "US-ASCII");
98    registrar("cp367", "US-ASCII");
99    registrar("csASCII", "US-ASCII");
100    registrar("ibm-367_P100-1995", "US-ASCII");
101    registrar("iso-ir-6", "US-ASCII");
102    registrar("iso-ir-6-us", "US-ASCII");
103    registrar("us", "US-ASCII");
104    registrar("x-ansi", "US-ASCII");
105}
106
107static PassOwnPtr<TextCodec> newStreamingTextDecoderWindowsLatin1(const TextEncoding&, const void*)
108{
109    return adoptPtr(new TextCodecLatin1);
110}
111
112void TextCodecLatin1::registerCodecs(TextCodecRegistrar registrar)
113{
114    registrar("windows-1252", newStreamingTextDecoderWindowsLatin1, 0);
115
116    // ASCII and Latin-1 both decode as Windows Latin-1 although they retain unique identities.
117    registrar("ISO-8859-1", newStreamingTextDecoderWindowsLatin1, 0);
118    registrar("US-ASCII", newStreamingTextDecoderWindowsLatin1, 0);
119}
120
121String TextCodecLatin1::decode(const char* bytes, size_t length, FlushBehavior, bool, bool&)
122{
123    LChar* characters;
124    if (!length)
125        return emptyString();
126    String result = String::createUninitialized(length, characters);
127
128    const uint8_t* source = reinterpret_cast<const uint8_t*>(bytes);
129    const uint8_t* end = reinterpret_cast<const uint8_t*>(bytes + length);
130    const uint8_t* alignedEnd = alignToMachineWord(end);
131    LChar* destination = characters;
132
133    while (source < end) {
134        if (isASCII(*source)) {
135            // Fast path for ASCII. Most Latin-1 text will be ASCII.
136            if (isAlignedToMachineWord(source)) {
137                while (source < alignedEnd) {
138                    MachineWord chunk = *reinterpret_cast_ptr<const MachineWord*>(source);
139
140                    if (!isAllASCII<LChar>(chunk))
141                        goto useLookupTable;
142
143                    copyASCIIMachineWord(destination, source);
144                    source += sizeof(MachineWord);
145                    destination += sizeof(MachineWord);
146                }
147
148                if (source == end)
149                    break;
150            }
151            *destination = *source;
152        } else {
153useLookupTable:
154            if (table[*source] > 0xff)
155                goto upConvertTo16Bit;
156
157            *destination = table[*source];
158        }
159
160        ++source;
161        ++destination;
162    }
163
164    return result;
165
166upConvertTo16Bit:
167    UChar* characters16;
168    String result16 = String::createUninitialized(length, characters16);
169
170    UChar* destination16 = characters16;
171
172    // Zero extend and copy already processed 8 bit data
173    LChar* ptr8 = characters;
174    LChar* endPtr8 = destination;
175
176    while (ptr8 < endPtr8)
177        *destination16++ = *ptr8++;
178
179    // Handle the character that triggered the 16 bit path
180    *destination16 = table[*source];
181    ++source;
182    ++destination16;
183
184    while (source < end) {
185        if (isASCII(*source)) {
186            // Fast path for ASCII. Most Latin-1 text will be ASCII.
187            if (isAlignedToMachineWord(source)) {
188                while (source < alignedEnd) {
189                    MachineWord chunk = *reinterpret_cast_ptr<const MachineWord*>(source);
190
191                    if (!isAllASCII<LChar>(chunk))
192                        goto useLookupTable16;
193
194                    copyASCIIMachineWord(destination16, source);
195                    source += sizeof(MachineWord);
196                    destination16 += sizeof(MachineWord);
197                }
198
199                if (source == end)
200                    break;
201            }
202            *destination16 = *source;
203        } else {
204useLookupTable16:
205            *destination16 = table[*source];
206        }
207
208        ++source;
209        ++destination16;
210    }
211
212    return result16;
213}
214
215template<typename CharType>
216static CString encodeComplexWindowsLatin1(const CharType* characters, size_t length, UnencodableHandling handling)
217{
218    Vector<char> result(length);
219    char* bytes = result.data();
220
221    size_t resultLength = 0;
222    for (size_t i = 0; i < length; ) {
223        UChar32 c;
224        U16_NEXT(characters, i, length, c);
225        unsigned char b = c;
226        // Do an efficient check to detect characters other than 00-7F and A0-FF.
227        if (b != c || (c & 0xE0) == 0x80) {
228            // Look for a way to encode this with Windows Latin-1.
229            for (b = 0x80; b < 0xA0; ++b)
230                if (table[b] == c)
231                    goto gotByte;
232            // No way to encode this character with Windows Latin-1.
233            UnencodableReplacementArray replacement;
234            int replacementLength = TextCodec::getUnencodableReplacement(c, handling, replacement);
235            result.grow(resultLength + replacementLength + length - i);
236            bytes = result.data();
237            memcpy(bytes + resultLength, replacement, replacementLength);
238            resultLength += replacementLength;
239            continue;
240        }
241    gotByte:
242        bytes[resultLength++] = b;
243    }
244
245    return CString(bytes, resultLength);
246}
247
248template<typename CharType>
249CString TextCodecLatin1::encodeCommon(const CharType* characters, size_t length, UnencodableHandling handling)
250{
251    {
252        char* bytes;
253        CString string = CString::newUninitialized(length, bytes);
254
255        // Convert the string a fast way and simultaneously do an efficient check to see if it's all ASCII.
256        UChar ored = 0;
257        for (size_t i = 0; i < length; ++i) {
258            UChar c = characters[i];
259            bytes[i] = c;
260            ored |= c;
261        }
262
263        if (!(ored & 0xFF80))
264            return string;
265    }
266
267    // If it wasn't all ASCII, call the function that handles more-complex cases.
268    return encodeComplexWindowsLatin1(characters, length, handling);
269}
270
271CString TextCodecLatin1::encode(const UChar* characters, size_t length, UnencodableHandling handling)
272{
273    return encodeCommon(characters, length, handling);
274}
275
276CString TextCodecLatin1::encode(const LChar* characters, size_t length, UnencodableHandling handling)
277{
278    return encodeCommon(characters, length, handling);
279}
280
281} // namespace WTF
282