icu_string_conversions_unittest.cc revision c2e0dbddbe15c98d52c4786dac06cb8952a8ae6d 
1// Copyright (c) 2011 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include <math.h>
6#include <stdarg.h>
7
8#include <limits>
9#include <sstream>
10
11#include "base/basictypes.h"
12#include "base/format_macros.h"
13#include "base/i18n/icu_string_conversions.h"
14#include "base/logging.h"
15#include "base/stringprintf.h"
16#include "base/strings/string_piece.h"
17#include "base/utf_string_conversions.h"
18#include "testing/gtest/include/gtest/gtest.h"
19
20namespace base {
21
22namespace {
23
24// Given a null-terminated string of wchar_t with each wchar_t representing
25// a UTF-16 code unit, returns a string16 made up of wchar_t's in the input.
26// Each wchar_t should be <= 0xFFFF and a non-BMP character (> U+FFFF)
27// should be represented as a surrogate pair (two UTF-16 units)
28// *even* where wchar_t is 32-bit (Linux and Mac).
29//
30// This is to help write tests for functions with string16 params until
31// the C++ 0x UTF-16 literal is well-supported by compilers.
32string16 BuildString16(const wchar_t* s) {
33#if defined(WCHAR_T_IS_UTF16)
34  return string16(s);
35#elif defined(WCHAR_T_IS_UTF32)
36  string16 u16;
37  while (*s != 0) {
38    DCHECK_LE(static_cast<unsigned int>(*s), 0xFFFFu);
39    u16.push_back(*s++);
40  }
41  return u16;
42#endif
43}
44
45const wchar_t* const kConvertRoundtripCases[] = {
46  L"Google Video",
47  // "网页 图片 资讯更多 »"
48  L"\x7f51\x9875\x0020\x56fe\x7247\x0020\x8d44\x8baf\x66f4\x591a\x0020\x00bb",
49  //  "Παγκόσμιος Ιστός"
50  L"\x03a0\x03b1\x03b3\x03ba\x03cc\x03c3\x03bc\x03b9"
51  L"\x03bf\x03c2\x0020\x0399\x03c3\x03c4\x03cc\x03c2",
52  // "Поиск страниц на русском"
53  L"\x041f\x043e\x0438\x0441\x043a\x0020\x0441\x0442"
54  L"\x0440\x0430\x043d\x0438\x0446\x0020\x043d\x0430"
55  L"\x0020\x0440\x0443\x0441\x0441\x043a\x043e\x043c",
56  // "전체서비스"
57  L"\xc804\xccb4\xc11c\xbe44\xc2a4",
58
59  // Test characters that take more than 16 bits. This will depend on whether
60  // wchar_t is 16 or 32 bits.
61#if defined(WCHAR_T_IS_UTF16)
62  L"\xd800\xdf00",
63  // ?????  (Mathematical Alphanumeric Symbols (U+011d40 - U+011d44 : A,B,C,D,E)
64  L"\xd807\xdd40\xd807\xdd41\xd807\xdd42\xd807\xdd43\xd807\xdd44",
65#elif defined(WCHAR_T_IS_UTF32)
66  L"\x10300",
67  // ?????  (Mathematical Alphanumeric Symbols (U+011d40 - U+011d44 : A,B,C,D,E)
68  L"\x11d40\x11d41\x11d42\x11d43\x11d44",
69#endif
70};
71
72}  // namespace
73
74TEST(ICUStringConversionsTest, ConvertCodepageUTF8) {
75  // Make sure WideToCodepage works like WideToUTF8.
76  for (size_t i = 0; i < arraysize(kConvertRoundtripCases); ++i) {
77    SCOPED_TRACE(base::StringPrintf("Test[%" PRIuS "]: %ls",
78                                    i, kConvertRoundtripCases[i]));
79
80    std::string expected(WideToUTF8(kConvertRoundtripCases[i]));
81    std::string utf8;
82    EXPECT_TRUE(WideToCodepage(kConvertRoundtripCases[i], kCodepageUTF8,
83                               OnStringConversionError::SKIP, &utf8));
84    EXPECT_EQ(expected, utf8);
85  }
86}
87
88// kConverterCodepageCases is not comprehensive. There are a number of cases
89// to add if we really want to have a comprehensive coverage of various
90// codepages and their 'idiosyncrasies'. Currently, the only implementation
91// for CodepageTo* and *ToCodepage uses ICU, which has a very extensive
92// set of tests for the charset conversion. So, we can get away with a
93// relatively small number of cases listed below.
94//
95// Note about |u16_wide| in the following struct.
96// On Windows, the field is always identical to |wide|. On Mac and Linux,
97// it's identical as long as there's no character outside the
98// BMP (<= U+FFFF). When there is, it is different from |wide| and
99// is not a real wide string (UTF-32 string) in that each wchar_t in
100// the string is a UTF-16 code unit zero-extended to be 32-bit
101// even when the code unit belongs to a surrogate pair.
102// For instance, a Unicode string (U+0041 U+010000) is represented as
103// L"\x0041\xD800\xDC00" instead of L"\x0041\x10000".
104// To avoid the clutter, |u16_wide| will be set to NULL
105// if it's identical to |wide| on *all* platforms.
106
107static const struct {
108  const char* codepage_name;
109  const char* encoded;
110  OnStringConversionError::Type on_error;
111  bool success;
112  const wchar_t* wide;
113  const wchar_t* u16_wide;
114} kConvertCodepageCases[] = {
115  // Test a case where the input cannot be decoded, using SKIP, FAIL
116  // and SUBSTITUTE error handling rules. "A7 41" is valid, but "A6" isn't.
117  {"big5",
118   "\xA7\x41\xA6",
119   OnStringConversionError::FAIL,
120   false,
121   L"",
122   NULL},
123  {"big5",
124   "\xA7\x41\xA6",
125   OnStringConversionError::SKIP,
126   true,
127   L"\x4F60",
128   NULL},
129  {"big5",
130   "\xA7\x41\xA6",
131   OnStringConversionError::SUBSTITUTE,
132   true,
133   L"\x4F60\xFFFD",
134   NULL},
135  // Arabic (ISO-8859)
136  {"iso-8859-6",
137   "\xC7\xEE\xE4\xD3\xF1\xEE\xE4\xC7\xE5\xEF" " "
138   "\xD9\xEE\xE4\xEE\xEA\xF2\xE3\xEF\xE5\xF2",
139   OnStringConversionError::FAIL,
140   true,
141   L"\x0627\x064E\x0644\x0633\x0651\x064E\x0644\x0627\x0645\x064F" L" "
142   L"\x0639\x064E\x0644\x064E\x064A\x0652\x0643\x064F\x0645\x0652",
143   NULL},
144  // Chinese Simplified (GB2312)
145  {"gb2312",
146   "\xC4\xE3\xBA\xC3",
147   OnStringConversionError::FAIL,
148   true,
149   L"\x4F60\x597D",
150   NULL},
151  // Chinese (GB18030) : 4 byte sequences mapped to BMP characters
152  {"gb18030",
153   "\x81\x30\x84\x36\xA1\xA7",
154   OnStringConversionError::FAIL,
155   true,
156   L"\x00A5\x00A8",
157   NULL},
158  // Chinese (GB18030) : A 4 byte sequence mapped to plane 2 (U+20000)
159  {"gb18030",
160   "\x95\x32\x82\x36\xD2\xBB",
161   OnStringConversionError::FAIL,
162   true,
163#if defined(WCHAR_T_IS_UTF16)
164   L"\xD840\xDC00\x4E00",
165#elif defined(WCHAR_T_IS_UTF32)
166   L"\x20000\x4E00",
167#endif
168   L"\xD840\xDC00\x4E00"},
169  {"big5",
170   "\xA7\x41\xA6\x6E",
171   OnStringConversionError::FAIL,
172   true,
173   L"\x4F60\x597D",
174   NULL},
175  // Greek (ISO-8859)
176  {"iso-8859-7",
177   "\xE3\xE5\xE9\xDC" " " "\xF3\xEF\xF5",
178   OnStringConversionError::FAIL,
179   true,
180   L"\x03B3\x03B5\x03B9\x03AC" L" " L"\x03C3\x03BF\x03C5",
181   NULL},
182  // Hebrew (Windows)
183  {"windows-1255",
184   "\xF9\xD1\xC8\xEC\xE5\xC9\xED",
185   OnStringConversionError::FAIL,
186   true,
187   L"\x05E9\x05C1\x05B8\x05DC\x05D5\x05B9\x05DD",
188   NULL},
189  // Hindi Devanagari (ISCII)
190  {"iscii-dev",
191   "\xEF\x42" "\xC6\xCC\xD7\xE8\xB3\xDA\xCF",
192   OnStringConversionError::FAIL,
193   true,
194   L"\x0928\x092E\x0938\x094D\x0915\x093E\x0930",
195   NULL},
196  // Korean (EUC)
197  {"euc-kr",
198   "\xBE\xC8\xB3\xE7\xC7\xCF\xBC\xBC\xBF\xE4",
199   OnStringConversionError::FAIL,
200   true,
201   L"\xC548\xB155\xD558\xC138\xC694",
202   NULL},
203  // Japanese (EUC)
204  {"euc-jp",
205   "\xA4\xB3\xA4\xF3\xA4\xCB\xA4\xC1\xA4\xCF\xB0\xEC\x8F\xB0\xA1\x8E\xA6",
206   OnStringConversionError::FAIL,
207   true,
208   L"\x3053\x3093\x306B\x3061\x306F\x4E00\x4E02\xFF66",
209   NULL},
210  // Japanese (ISO-2022)
211  {"iso-2022-jp",
212   "\x1B$B" "\x24\x33\x24\x73\x24\x4B\x24\x41\x24\x4F\x30\x6C" "\x1B(B"
213   "ab" "\x1B(J" "\x5C\x7E#$" "\x1B(B",
214   OnStringConversionError::FAIL,
215   true,
216   L"\x3053\x3093\x306B\x3061\x306F\x4E00" L"ab\x00A5\x203E#$",
217   NULL},
218  // Japanese (Shift-JIS)
219  {"sjis",
220   "\x82\xB1\x82\xF1\x82\xC9\x82\xBF\x82\xCD\x88\xEA\xA6",
221   OnStringConversionError::FAIL,
222   true,
223   L"\x3053\x3093\x306B\x3061\x306F\x4E00\xFF66",
224   NULL},
225  // Russian (KOI8)
226  {"koi8-r",
227   "\xDA\xC4\xD2\xC1\xD7\xD3\xD4\xD7\xD5\xCA\xD4\xC5",
228   OnStringConversionError::FAIL,
229   true,
230   L"\x0437\x0434\x0440\x0430\x0432\x0441\x0442\x0432"
231   L"\x0443\x0439\x0442\x0435",
232   NULL},
233  // Thai (windows-874)
234  {"windows-874",
235   "\xCA\xC7\xD1\xCA\xB4\xD5" "\xA4\xC3\xD1\xBA",
236   OnStringConversionError::FAIL,
237   true,
238   L"\x0E2A\x0E27\x0E31\x0E2A\x0E14\x0E35"
239   L"\x0E04\x0E23\x0e31\x0E1A",
240   NULL},
241  // Empty text
242  {"iscii-dev",
243   "",
244   OnStringConversionError::FAIL,
245   true,
246   L"",
247   NULL},
248};
249
250TEST(ICUStringConversionsTest, ConvertBetweenCodepageAndWide) {
251  for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kConvertCodepageCases); ++i) {
252    SCOPED_TRACE(base::StringPrintf(
253                     "Test[%" PRIuS "]: <encoded: %s> <codepage: %s>", i,
254                     kConvertCodepageCases[i].encoded,
255                     kConvertCodepageCases[i].codepage_name));
256
257    std::wstring wide;
258    bool success = CodepageToWide(kConvertCodepageCases[i].encoded,
259                                  kConvertCodepageCases[i].codepage_name,
260                                  kConvertCodepageCases[i].on_error,
261                                  &wide);
262    EXPECT_EQ(kConvertCodepageCases[i].success, success);
263    EXPECT_EQ(kConvertCodepageCases[i].wide, wide);
264
265    // When decoding was successful and nothing was skipped, we also check the
266    // reverse conversion. Not all conversions are round-trippable, but
267    // kConverterCodepageCases does not have any one-way conversion at the
268    // moment.
269    if (success &&
270        kConvertCodepageCases[i].on_error ==
271            OnStringConversionError::FAIL) {
272      std::string encoded;
273      success = WideToCodepage(wide, kConvertCodepageCases[i].codepage_name,
274                               kConvertCodepageCases[i].on_error, &encoded);
275      EXPECT_EQ(kConvertCodepageCases[i].success, success);
276      EXPECT_EQ(kConvertCodepageCases[i].encoded, encoded);
277    }
278  }
279
280  // The above cases handled codepage->wide errors, but not wide->codepage.
281  // Test that here.
282  std::string encoded("Temp data");  // Make sure the string gets cleared.
283
284  // First test going to an encoding that can not represent that character.
285  EXPECT_FALSE(WideToCodepage(L"Chinese\xff27", "iso-8859-1",
286                              OnStringConversionError::FAIL, &encoded));
287  EXPECT_TRUE(encoded.empty());
288  EXPECT_TRUE(WideToCodepage(L"Chinese\xff27", "iso-8859-1",
289                             OnStringConversionError::SKIP, &encoded));
290  EXPECT_STREQ("Chinese", encoded.c_str());
291  // From Unicode, SUBSTITUTE is the same as SKIP for now.
292  EXPECT_TRUE(WideToCodepage(L"Chinese\xff27", "iso-8859-1",
293                             OnStringConversionError::SUBSTITUTE,
294                             &encoded));
295  EXPECT_STREQ("Chinese", encoded.c_str());
296
297#if defined(WCHAR_T_IS_UTF16)
298  // When we're in UTF-16 mode, test an invalid UTF-16 character in the input.
299  EXPECT_FALSE(WideToCodepage(L"a\xd800z", "iso-8859-1",
300                              OnStringConversionError::FAIL, &encoded));
301  EXPECT_TRUE(encoded.empty());
302  EXPECT_TRUE(WideToCodepage(L"a\xd800z", "iso-8859-1",
303                             OnStringConversionError::SKIP, &encoded));
304  EXPECT_STREQ("az", encoded.c_str());
305#endif  // WCHAR_T_IS_UTF16
306
307  // Invalid characters should fail.
308  EXPECT_TRUE(WideToCodepage(L"a\xffffz", "iso-8859-1",
309                             OnStringConversionError::SKIP, &encoded));
310  EXPECT_STREQ("az", encoded.c_str());
311
312  // Invalid codepages should fail.
313  EXPECT_FALSE(WideToCodepage(L"Hello, world", "awesome-8571-2",
314                              OnStringConversionError::SKIP, &encoded));
315}
316
317TEST(ICUStringConversionsTest, ConvertBetweenCodepageAndUTF16) {
318  for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kConvertCodepageCases); ++i) {
319    SCOPED_TRACE(base::StringPrintf(
320                     "Test[%" PRIuS "]: <encoded: %s> <codepage: %s>", i,
321                     kConvertCodepageCases[i].encoded,
322                     kConvertCodepageCases[i].codepage_name));
323
324    string16 utf16;
325    bool success = CodepageToUTF16(kConvertCodepageCases[i].encoded,
326                                   kConvertCodepageCases[i].codepage_name,
327                                   kConvertCodepageCases[i].on_error,
328                                   &utf16);
329    string16 utf16_expected;
330    if (kConvertCodepageCases[i].u16_wide == NULL)
331      utf16_expected = BuildString16(kConvertCodepageCases[i].wide);
332    else
333      utf16_expected = BuildString16(kConvertCodepageCases[i].u16_wide);
334    EXPECT_EQ(kConvertCodepageCases[i].success, success);
335    EXPECT_EQ(utf16_expected, utf16);
336
337    // When decoding was successful and nothing was skipped, we also check the
338    // reverse conversion. See also the corresponding comment in
339    // ConvertBetweenCodepageAndWide.
340    if (success &&
341        kConvertCodepageCases[i].on_error == OnStringConversionError::FAIL) {
342      std::string encoded;
343      success = UTF16ToCodepage(utf16, kConvertCodepageCases[i].codepage_name,
344                                kConvertCodepageCases[i].on_error, &encoded);
345      EXPECT_EQ(kConvertCodepageCases[i].success, success);
346      EXPECT_EQ(kConvertCodepageCases[i].encoded, encoded);
347    }
348  }
349}
350
351static const struct {
352  const char* encoded;
353  const char* codepage_name;
354  bool expected_success;
355  const char* expected_value;
356} kConvertAndNormalizeCases[] = {
357  {"foo-\xe4.html", "iso-8859-1", true, "foo-\xc3\xa4.html"},
358  {"foo-\xe4.html", "iso-8859-7", true, "foo-\xce\xb4.html"},
359  {"foo-\xe4.html", "foo-bar", false, ""},
360  {"foo-\xff.html", "ascii", false, ""},
361  {"foo.html", "ascii", true, "foo.html"},
362  {"foo-a\xcc\x88.html", "utf-8", true, "foo-\xc3\xa4.html"},
363  {"\x95\x32\x82\x36\xD2\xBB", "gb18030", true, "\xF0\xA0\x80\x80\xE4\xB8\x80"},
364  {"\xA7\x41\xA6\x6E", "big5", true, "\xE4\xBD\xA0\xE5\xA5\xBD"},
365  // Windows-1258 does have a combining character at xD2 (which is U+0309).
366  // The sequence of (U+00E2, U+0309) is also encoded as U+1EA9.
367  {"foo\xE2\xD2", "windows-1258", true, "foo\xE1\xBA\xA9"},
368  {"", "iso-8859-1", true, ""},
369};
370TEST(ICUStringConversionsTest, ConvertToUtf8AndNormalize) {
371  std::string result;
372  for (size_t i = 0; i < ARRAYSIZE_UNSAFE(kConvertAndNormalizeCases); ++i) {
373    SCOPED_TRACE(base::StringPrintf(
374                     "Test[%" PRIuS "]: <encoded: %s> <codepage: %s>", i,
375                     kConvertAndNormalizeCases[i].encoded,
376                     kConvertAndNormalizeCases[i].codepage_name));
377
378    bool success = ConvertToUtf8AndNormalize(
379        kConvertAndNormalizeCases[i].encoded,
380        kConvertAndNormalizeCases[i].codepage_name, &result);
381    EXPECT_EQ(kConvertAndNormalizeCases[i].expected_success, success);
382    EXPECT_EQ(kConvertAndNormalizeCases[i].expected_value, result);
383  }
384}
385
386}  // namespace base
387