icu_string_conversions.cc revision 513209b27ff55e2841eac0e4120199c23acce758 
1// Copyright (c) 2009 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "base/i18n/icu_string_conversions.h"
6
7#include <vector>
8
9#include "base/basictypes.h"
10#include "base/logging.h"
11#include "base/string_util.h"
12#include "base/utf_string_conversions.h"
13#include "unicode/ucnv.h"
14#include "unicode/ucnv_cb.h"
15#include "unicode/ucnv_err.h"
16#include "unicode/unorm.h"
17#include "unicode/ustring.h"
18
19namespace base {
20
21namespace {
22// ToUnicodeCallbackSubstitute() is based on UCNV_TO_U_CALLBACK_SUSBSTITUTE
23// in source/common/ucnv_err.c.
24
25// Copyright (c) 1995-2006 International Business Machines Corporation
26// and others
27//
28// All rights reserved.
29//
30
31// Permission is hereby granted, free of charge, to any person obtaining a
32// copy of this software and associated documentation files (the "Software"),
33// to deal in the Software without restriction, including without limitation
34// the rights to use, copy, modify, merge, publish, distribute, and/or
35// sell copies of the Software, and to permit persons to whom the Software
36// is furnished to do so, provided that the above copyright notice(s) and
37// this permission notice appear in all copies of the Software and that
38// both the above copyright notice(s) and this permission notice appear in
39// supporting documentation.
40//
41// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
42// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
43// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
44// OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
45// INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT
46// OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
47// OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
48// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE
49// OR PERFORMANCE OF THIS SOFTWARE.
50//
51// Except as contained in this notice, the name of a copyright holder
52// shall not be used in advertising or otherwise to promote the sale, use
53// or other dealings in this Software without prior written authorization
54// of the copyright holder.
55
56//  ___________________________________________________________________________
57//
58// All trademarks and registered trademarks mentioned herein are the property
59// of their respective owners.
60
61void ToUnicodeCallbackSubstitute(const void* context,
62                                 UConverterToUnicodeArgs *to_args,
63                                 const char* code_units,
64                                 int32_t length,
65                                 UConverterCallbackReason reason,
66                                 UErrorCode * err) {
67  static const UChar kReplacementChar = 0xFFFD;
68  if (reason <= UCNV_IRREGULAR) {
69      if (context == NULL ||
70          (*(reinterpret_cast<const char*>(context)) == 'i' &&
71           reason == UCNV_UNASSIGNED)) {
72        *err = U_ZERO_ERROR;
73        ucnv_cbToUWriteUChars(to_args, &kReplacementChar, 1, 0, err);
74      }
75      // else the caller must have set the error code accordingly.
76  }
77  // else ignore the reset, close and clone calls.
78}
79
80bool ConvertFromUTF16(UConverter* converter, const UChar* uchar_src,
81                      int uchar_len, OnStringConversionError::Type on_error,
82                      std::string* encoded) {
83  int encoded_max_length = UCNV_GET_MAX_BYTES_FOR_STRING(uchar_len,
84      ucnv_getMaxCharSize(converter));
85  encoded->resize(encoded_max_length);
86
87  UErrorCode status = U_ZERO_ERROR;
88
89  // Setup our error handler.
90  switch (on_error) {
91    case OnStringConversionError::FAIL:
92      ucnv_setFromUCallBack(converter, UCNV_FROM_U_CALLBACK_STOP, 0,
93                            NULL, NULL, &status);
94      break;
95    case OnStringConversionError::SKIP:
96    case OnStringConversionError::SUBSTITUTE:
97      ucnv_setFromUCallBack(converter, UCNV_FROM_U_CALLBACK_SKIP, 0,
98                            NULL, NULL, &status);
99      break;
100    default:
101      NOTREACHED();
102  }
103
104  // ucnv_fromUChars returns size not including terminating null
105  int actual_size = ucnv_fromUChars(converter, &(*encoded)[0],
106      encoded_max_length, uchar_src, uchar_len, &status);
107  encoded->resize(actual_size);
108  ucnv_close(converter);
109  if (U_SUCCESS(status))
110    return true;
111  encoded->clear();  // Make sure the output is empty on error.
112  return false;
113}
114
115// Set up our error handler for ToUTF-16 converters
116void SetUpErrorHandlerForToUChars(OnStringConversionError::Type on_error,
117                                  UConverter* converter, UErrorCode* status) {
118  switch (on_error) {
119    case OnStringConversionError::FAIL:
120      ucnv_setToUCallBack(converter, UCNV_TO_U_CALLBACK_STOP, 0,
121                          NULL, NULL, status);
122      break;
123    case OnStringConversionError::SKIP:
124      ucnv_setToUCallBack(converter, UCNV_TO_U_CALLBACK_SKIP, 0,
125                          NULL, NULL, status);
126      break;
127    case OnStringConversionError::SUBSTITUTE:
128      ucnv_setToUCallBack(converter, ToUnicodeCallbackSubstitute, 0,
129                          NULL, NULL, status);
130      break;
131    default:
132      NOTREACHED();
133  }
134}
135
136inline UConverterType utf32_platform_endian() {
137#if U_IS_BIG_ENDIAN
138  return UCNV_UTF32_BigEndian;
139#else
140  return UCNV_UTF32_LittleEndian;
141#endif
142}
143
144}  // namespace
145
146const char kCodepageLatin1[] = "ISO-8859-1";
147const char kCodepageUTF8[] = "UTF-8";
148const char kCodepageUTF16BE[] = "UTF-16BE";
149const char kCodepageUTF16LE[] = "UTF-16LE";
150
151// Codepage <-> Wide/UTF-16  ---------------------------------------------------
152
153bool UTF16ToCodepage(const string16& utf16,
154                     const char* codepage_name,
155                     OnStringConversionError::Type on_error,
156                     std::string* encoded) {
157  encoded->clear();
158
159  UErrorCode status = U_ZERO_ERROR;
160  UConverter* converter = ucnv_open(codepage_name, &status);
161  if (!U_SUCCESS(status))
162    return false;
163
164  return ConvertFromUTF16(converter, utf16.c_str(),
165                          static_cast<int>(utf16.length()), on_error, encoded);
166}
167
168bool CodepageToUTF16(const std::string& encoded,
169                     const char* codepage_name,
170                     OnStringConversionError::Type on_error,
171                     string16* utf16) {
172  utf16->clear();
173
174  UErrorCode status = U_ZERO_ERROR;
175  UConverter* converter = ucnv_open(codepage_name, &status);
176  if (!U_SUCCESS(status))
177    return false;
178
179  // Even in the worst case, the maximum length in 2-byte units of UTF-16
180  // output would be at most the same as the number of bytes in input. There
181  // is no single-byte encoding in which a character is mapped to a
182  // non-BMP character requiring two 2-byte units.
183  //
184  // Moreover, non-BMP characters in legacy multibyte encodings
185  // (e.g. EUC-JP, GB18030) take at least 2 bytes. The only exceptions are
186  // BOCU and SCSU, but we don't care about them.
187  size_t uchar_max_length = encoded.length() + 1;
188
189  SetUpErrorHandlerForToUChars(on_error, converter, &status);
190  int actual_size = ucnv_toUChars(converter, WriteInto(utf16, uchar_max_length),
191      static_cast<int>(uchar_max_length), encoded.data(),
192      static_cast<int>(encoded.length()), &status);
193  ucnv_close(converter);
194  if (!U_SUCCESS(status)) {
195    utf16->clear();  // Make sure the output is empty on error.
196    return false;
197  }
198
199  utf16->resize(actual_size);
200  return true;
201}
202
203bool WideToCodepage(const std::wstring& wide,
204                    const char* codepage_name,
205                    OnStringConversionError::Type on_error,
206                    std::string* encoded) {
207#if defined(WCHAR_T_IS_UTF16)
208  return UTF16ToCodepage(wide, codepage_name, on_error, encoded);
209#elif defined(WCHAR_T_IS_UTF32)
210  encoded->clear();
211
212  UErrorCode status = U_ZERO_ERROR;
213  UConverter* converter = ucnv_open(codepage_name, &status);
214  if (!U_SUCCESS(status))
215    return false;
216
217  int utf16_len;
218  // When wchar_t is wider than UChar (16 bits), transform |wide| into a
219  // UChar* string.  Size the UChar* buffer to be large enough to hold twice
220  // as many UTF-16 code units (UChar's) as there are Unicode code points,
221  // in case each code points translates to a UTF-16 surrogate pair,
222  // and leave room for a NUL terminator.
223  std::vector<UChar> utf16(wide.length() * 2 + 1);
224  u_strFromWCS(&utf16[0], utf16.size(), &utf16_len,
225               wide.c_str(), wide.length(), &status);
226  DCHECK(U_SUCCESS(status)) << "failed to convert wstring to UChar*";
227
228  return ConvertFromUTF16(converter, &utf16[0], utf16_len, on_error, encoded);
229#endif  // defined(WCHAR_T_IS_UTF32)
230}
231
232bool CodepageToWide(const std::string& encoded,
233                    const char* codepage_name,
234                    OnStringConversionError::Type on_error,
235                    std::wstring* wide) {
236#if defined(WCHAR_T_IS_UTF16)
237  return CodepageToUTF16(encoded, codepage_name, on_error, wide);
238#elif defined(WCHAR_T_IS_UTF32)
239  wide->clear();
240
241  UErrorCode status = U_ZERO_ERROR;
242  UConverter* converter = ucnv_open(codepage_name, &status);
243  if (!U_SUCCESS(status))
244    return false;
245
246  // The maximum length in 4 byte unit of UTF-32 output would be
247  // at most the same as the number of bytes in input. In the worst
248  // case of GB18030 (excluding escaped-based encodings like ISO-2022-JP),
249  // this can be 4 times larger than actually needed.
250  size_t wchar_max_length = encoded.length() + 1;
251
252  SetUpErrorHandlerForToUChars(on_error, converter, &status);
253  int actual_size = ucnv_toAlgorithmic(utf32_platform_endian(), converter,
254      reinterpret_cast<char*>(WriteInto(wide, wchar_max_length)),
255      static_cast<int>(wchar_max_length) * sizeof(wchar_t), encoded.data(),
256      static_cast<int>(encoded.length()), &status);
257  ucnv_close(converter);
258  if (!U_SUCCESS(status)) {
259    wide->clear();  // Make sure the output is empty on error.
260    return false;
261  }
262
263  // actual_size is # of bytes.
264  wide->resize(actual_size / sizeof(wchar_t));
265  return true;
266#endif  // defined(WCHAR_T_IS_UTF32)
267}
268
269bool ConvertToUtf8AndNormalize(const std::string& text,
270                               const std::string& charset,
271                               std::string* result) {
272  result->clear();
273  string16 utf16;
274  if (!CodepageToUTF16(
275      text, charset.c_str(), OnStringConversionError::FAIL, &utf16))
276    return false;
277
278  UErrorCode status = U_ZERO_ERROR;
279  size_t max_length = utf16.length() + 1;
280  string16 normalized_utf16;
281  int actual_length = unorm_normalize(
282      utf16.c_str(), utf16.length(), UNORM_NFC, 0,
283      WriteInto(&normalized_utf16, max_length),
284      static_cast<int>(max_length), &status);
285  if (!U_SUCCESS(status))
286    return false;
287  normalized_utf16.resize(actual_length);
288
289  return UTF16ToUTF8(normalized_utf16.data(),
290                     normalized_utf16.length(), result);
291}
292
293}  // namespace base
294