1// Copyright (c) 2011 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include <string>
6
7#include "base/basictypes.h"
8#include "base/strings/string_piece.h"
9#include "base/strings/sys_string_conversions.h"
10#include "base/strings/utf_string_conversions.h"
11#include "base/test/scoped_locale.h"
12#include "testing/gtest/include/gtest/gtest.h"
13
14#ifdef WCHAR_T_IS_UTF32
15static const std::wstring kSysWideOldItalicLetterA = L"\x10300";
16#else
17static const std::wstring kSysWideOldItalicLetterA = L"\xd800\xdf00";
18#endif
19
20namespace base {
21
22TEST(SysStrings, SysWideToUTF8) {
23  EXPECT_EQ("Hello, world", SysWideToUTF8(L"Hello, world"));
24  EXPECT_EQ("\xe4\xbd\xa0\xe5\xa5\xbd", SysWideToUTF8(L"\x4f60\x597d"));
25
26  // >16 bits
27  EXPECT_EQ("\xF0\x90\x8C\x80", SysWideToUTF8(kSysWideOldItalicLetterA));
28
29  // Error case. When Windows finds a UTF-16 character going off the end of
30  // a string, it just converts that literal value to UTF-8, even though this
31  // is invalid.
32  //
33  // This is what XP does, but Vista has different behavior, so we don't bother
34  // verifying it:
35  // EXPECT_EQ("\xE4\xBD\xA0\xED\xA0\x80zyxw",
36  //           SysWideToUTF8(L"\x4f60\xd800zyxw"));
37
38  // Test embedded NULLs.
39  std::wstring wide_null(L"a");
40  wide_null.push_back(0);
41  wide_null.push_back('b');
42
43  std::string expected_null("a");
44  expected_null.push_back(0);
45  expected_null.push_back('b');
46
47  EXPECT_EQ(expected_null, SysWideToUTF8(wide_null));
48}
49
50TEST(SysStrings, SysUTF8ToWide) {
51  EXPECT_EQ(L"Hello, world", SysUTF8ToWide("Hello, world"));
52  EXPECT_EQ(L"\x4f60\x597d", SysUTF8ToWide("\xe4\xbd\xa0\xe5\xa5\xbd"));
53  // >16 bits
54  EXPECT_EQ(kSysWideOldItalicLetterA, SysUTF8ToWide("\xF0\x90\x8C\x80"));
55
56  // Error case. When Windows finds an invalid UTF-8 character, it just skips
57  // it. This seems weird because it's inconsistent with the reverse conversion.
58  //
59  // This is what XP does, but Vista has different behavior, so we don't bother
60  // verifying it:
61  // EXPECT_EQ(L"\x4f60zyxw", SysUTF8ToWide("\xe4\xbd\xa0\xe5\xa5zyxw"));
62
63  // Test embedded NULLs.
64  std::string utf8_null("a");
65  utf8_null.push_back(0);
66  utf8_null.push_back('b');
67
68  std::wstring expected_null(L"a");
69  expected_null.push_back(0);
70  expected_null.push_back('b');
71
72  EXPECT_EQ(expected_null, SysUTF8ToWide(utf8_null));
73}
74
75#if defined(OS_LINUX)  // Tests depend on setting a specific Linux locale.
76
77TEST(SysStrings, SysWideToNativeMB) {
78  ScopedLocale locale("en_US.utf-8");
79  EXPECT_EQ("Hello, world", SysWideToNativeMB(L"Hello, world"));
80  EXPECT_EQ("\xe4\xbd\xa0\xe5\xa5\xbd", SysWideToNativeMB(L"\x4f60\x597d"));
81
82  // >16 bits
83  EXPECT_EQ("\xF0\x90\x8C\x80", SysWideToNativeMB(kSysWideOldItalicLetterA));
84
85  // Error case. When Windows finds a UTF-16 character going off the end of
86  // a string, it just converts that literal value to UTF-8, even though this
87  // is invalid.
88  //
89  // This is what XP does, but Vista has different behavior, so we don't bother
90  // verifying it:
91  // EXPECT_EQ("\xE4\xBD\xA0\xED\xA0\x80zyxw",
92  //           SysWideToNativeMB(L"\x4f60\xd800zyxw"));
93
94  // Test embedded NULLs.
95  std::wstring wide_null(L"a");
96  wide_null.push_back(0);
97  wide_null.push_back('b');
98
99  std::string expected_null("a");
100  expected_null.push_back(0);
101  expected_null.push_back('b');
102
103  EXPECT_EQ(expected_null, SysWideToNativeMB(wide_null));
104}
105
106// We assume the test is running in a UTF8 locale.
107TEST(SysStrings, SysNativeMBToWide) {
108  ScopedLocale locale("en_US.utf-8");
109  EXPECT_EQ(L"Hello, world", SysNativeMBToWide("Hello, world"));
110  EXPECT_EQ(L"\x4f60\x597d", SysNativeMBToWide("\xe4\xbd\xa0\xe5\xa5\xbd"));
111  // >16 bits
112  EXPECT_EQ(kSysWideOldItalicLetterA, SysNativeMBToWide("\xF0\x90\x8C\x80"));
113
114  // Error case. When Windows finds an invalid UTF-8 character, it just skips
115  // it. This seems weird because it's inconsistent with the reverse conversion.
116  //
117  // This is what XP does, but Vista has different behavior, so we don't bother
118  // verifying it:
119  // EXPECT_EQ(L"\x4f60zyxw", SysNativeMBToWide("\xe4\xbd\xa0\xe5\xa5zyxw"));
120
121  // Test embedded NULLs.
122  std::string utf8_null("a");
123  utf8_null.push_back(0);
124  utf8_null.push_back('b');
125
126  std::wstring expected_null(L"a");
127  expected_null.push_back(0);
128  expected_null.push_back('b');
129
130  EXPECT_EQ(expected_null, SysNativeMBToWide(utf8_null));
131}
132
133static const wchar_t* const kConvertRoundtripCases[] = {
134  L"Google Video",
135  // "网页 图片 资讯更多 »"
136  L"\x7f51\x9875\x0020\x56fe\x7247\x0020\x8d44\x8baf\x66f4\x591a\x0020\x00bb",
137  //  "Παγκόσμιος Ιστός"
138  L"\x03a0\x03b1\x03b3\x03ba\x03cc\x03c3\x03bc\x03b9"
139  L"\x03bf\x03c2\x0020\x0399\x03c3\x03c4\x03cc\x03c2",
140  // "Поиск страниц на русском"
141  L"\x041f\x043e\x0438\x0441\x043a\x0020\x0441\x0442"
142  L"\x0440\x0430\x043d\x0438\x0446\x0020\x043d\x0430"
143  L"\x0020\x0440\x0443\x0441\x0441\x043a\x043e\x043c",
144  // "전체서비스"
145  L"\xc804\xccb4\xc11c\xbe44\xc2a4",
146
147  // Test characters that take more than 16 bits. This will depend on whether
148  // wchar_t is 16 or 32 bits.
149#if defined(WCHAR_T_IS_UTF16)
150  L"\xd800\xdf00",
151  // ?????  (Mathematical Alphanumeric Symbols (U+011d40 - U+011d44 : A,B,C,D,E)
152  L"\xd807\xdd40\xd807\xdd41\xd807\xdd42\xd807\xdd43\xd807\xdd44",
153#elif defined(WCHAR_T_IS_UTF32)
154  L"\x10300",
155  // ?????  (Mathematical Alphanumeric Symbols (U+011d40 - U+011d44 : A,B,C,D,E)
156  L"\x11d40\x11d41\x11d42\x11d43\x11d44",
157#endif
158};
159
160
161TEST(SysStrings, SysNativeMBAndWide) {
162  ScopedLocale locale("en_US.utf-8");
163  for (size_t i = 0; i < arraysize(kConvertRoundtripCases); ++i) {
164    std::wstring wide = kConvertRoundtripCases[i];
165    std::wstring trip = SysNativeMBToWide(SysWideToNativeMB(wide));
166    EXPECT_EQ(wide.size(), trip.size());
167    EXPECT_EQ(wide, trip);
168  }
169
170  // We assume our test is running in UTF-8, so double check through ICU.
171  for (size_t i = 0; i < arraysize(kConvertRoundtripCases); ++i) {
172    std::wstring wide = kConvertRoundtripCases[i];
173    std::wstring trip = SysNativeMBToWide(WideToUTF8(wide));
174    EXPECT_EQ(wide.size(), trip.size());
175    EXPECT_EQ(wide, trip);
176  }
177
178  for (size_t i = 0; i < arraysize(kConvertRoundtripCases); ++i) {
179    std::wstring wide = kConvertRoundtripCases[i];
180    std::wstring trip = UTF8ToWide(SysWideToNativeMB(wide));
181    EXPECT_EQ(wide.size(), trip.size());
182    EXPECT_EQ(wide, trip);
183  }
184}
185#endif  // OS_LINUX
186
187}  // namespace base
188