icu_string_conversions.cc revision 2a99a7e74a7f215066514fe81d2bfa6639d9eddd 
1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "base/i18n/icu_string_conversions.h"
6
7#include <vector>
8
9#include "base/basictypes.h"
10#include "base/logging.h"
11#include "base/memory/scoped_ptr.h"
12#include "base/string_util.h"
13#include "base/utf_string_conversions.h"
14#include "third_party/icu/public/common/unicode/ucnv.h"
15#include "third_party/icu/public/common/unicode/ucnv_cb.h"
16#include "third_party/icu/public/common/unicode/ucnv_err.h"
17#include "third_party/icu/public/common/unicode/unorm.h"
18#include "third_party/icu/public/common/unicode/ustring.h"
19
20namespace base {
21
22namespace {
23// ToUnicodeCallbackSubstitute() is based on UCNV_TO_U_CALLBACK_SUBSTITUTE
24// in source/common/ucnv_err.c.
25
26// Copyright (c) 1995-2006 International Business Machines Corporation
27// and others
28//
29// All rights reserved.
30//
31
32// Permission is hereby granted, free of charge, to any person obtaining a
33// copy of this software and associated documentation files (the "Software"),
34// to deal in the Software without restriction, including without limitation
35// the rights to use, copy, modify, merge, publish, distribute, and/or
36// sell copies of the Software, and to permit persons to whom the Software
37// is furnished to do so, provided that the above copyright notice(s) and
38// this permission notice appear in all copies of the Software and that
39// both the above copyright notice(s) and this permission notice appear in
40// supporting documentation.
41//
42// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
43// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
44// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
45// OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS
46// INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT
47// OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS
48// OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE
49// OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE
50// OR PERFORMANCE OF THIS SOFTWARE.
51//
52// Except as contained in this notice, the name of a copyright holder
53// shall not be used in advertising or otherwise to promote the sale, use
54// or other dealings in this Software without prior written authorization
55// of the copyright holder.
56
57//  ___________________________________________________________________________
58//
59// All trademarks and registered trademarks mentioned herein are the property
60// of their respective owners.
61
62void ToUnicodeCallbackSubstitute(const void* context,
63                                 UConverterToUnicodeArgs *to_args,
64                                 const char* code_units,
65                                 int32_t length,
66                                 UConverterCallbackReason reason,
67                                 UErrorCode * err) {
68  static const UChar kReplacementChar = 0xFFFD;
69  if (reason <= UCNV_IRREGULAR) {
70      if (context == NULL ||
71          (*(reinterpret_cast<const char*>(context)) == 'i' &&
72           reason == UCNV_UNASSIGNED)) {
73        *err = U_ZERO_ERROR;
74        ucnv_cbToUWriteUChars(to_args, &kReplacementChar, 1, 0, err);
75      }
76      // else the caller must have set the error code accordingly.
77  }
78  // else ignore the reset, close and clone calls.
79}
80
81bool ConvertFromUTF16(UConverter* converter, const UChar* uchar_src,
82                      int uchar_len, OnStringConversionError::Type on_error,
83                      std::string* encoded) {
84  int encoded_max_length = UCNV_GET_MAX_BYTES_FOR_STRING(uchar_len,
85      ucnv_getMaxCharSize(converter));
86  encoded->resize(encoded_max_length);
87
88  UErrorCode status = U_ZERO_ERROR;
89
90  // Setup our error handler.
91  switch (on_error) {
92    case OnStringConversionError::FAIL:
93      ucnv_setFromUCallBack(converter, UCNV_FROM_U_CALLBACK_STOP, 0,
94                            NULL, NULL, &status);
95      break;
96    case OnStringConversionError::SKIP:
97    case OnStringConversionError::SUBSTITUTE:
98      ucnv_setFromUCallBack(converter, UCNV_FROM_U_CALLBACK_SKIP, 0,
99                            NULL, NULL, &status);
100      break;
101    default:
102      NOTREACHED();
103  }
104
105  // ucnv_fromUChars returns size not including terminating null
106  int actual_size = ucnv_fromUChars(converter, &(*encoded)[0],
107      encoded_max_length, uchar_src, uchar_len, &status);
108  encoded->resize(actual_size);
109  ucnv_close(converter);
110  if (U_SUCCESS(status))
111    return true;
112  encoded->clear();  // Make sure the output is empty on error.
113  return false;
114}
115
116// Set up our error handler for ToUTF-16 converters
117void SetUpErrorHandlerForToUChars(OnStringConversionError::Type on_error,
118                                  UConverter* converter, UErrorCode* status) {
119  switch (on_error) {
120    case OnStringConversionError::FAIL:
121      ucnv_setToUCallBack(converter, UCNV_TO_U_CALLBACK_STOP, 0,
122                          NULL, NULL, status);
123      break;
124    case OnStringConversionError::SKIP:
125      ucnv_setToUCallBack(converter, UCNV_TO_U_CALLBACK_SKIP, 0,
126                          NULL, NULL, status);
127      break;
128    case OnStringConversionError::SUBSTITUTE:
129      ucnv_setToUCallBack(converter, ToUnicodeCallbackSubstitute, 0,
130                          NULL, NULL, status);
131      break;
132    default:
133      NOTREACHED();
134  }
135}
136
137inline UConverterType utf32_platform_endian() {
138#if U_IS_BIG_ENDIAN
139  return UCNV_UTF32_BigEndian;
140#else
141  return UCNV_UTF32_LittleEndian;
142#endif
143}
144
145}  // namespace
146
147const char kCodepageLatin1[] = "ISO-8859-1";
148const char kCodepageUTF8[] = "UTF-8";
149const char kCodepageUTF16BE[] = "UTF-16BE";
150const char kCodepageUTF16LE[] = "UTF-16LE";
151
152// Codepage <-> Wide/UTF-16  ---------------------------------------------------
153
154bool UTF16ToCodepage(const string16& utf16,
155                     const char* codepage_name,
156                     OnStringConversionError::Type on_error,
157                     std::string* encoded) {
158  encoded->clear();
159
160  UErrorCode status = U_ZERO_ERROR;
161  UConverter* converter = ucnv_open(codepage_name, &status);
162  if (!U_SUCCESS(status))
163    return false;
164
165  return ConvertFromUTF16(converter, utf16.c_str(),
166                          static_cast<int>(utf16.length()), on_error, encoded);
167}
168
169bool CodepageToUTF16(const std::string& encoded,
170                     const char* codepage_name,
171                     OnStringConversionError::Type on_error,
172                     string16* utf16) {
173  utf16->clear();
174
175  UErrorCode status = U_ZERO_ERROR;
176  UConverter* converter = ucnv_open(codepage_name, &status);
177  if (!U_SUCCESS(status))
178    return false;
179
180  // Even in the worst case, the maximum length in 2-byte units of UTF-16
181  // output would be at most the same as the number of bytes in input. There
182  // is no single-byte encoding in which a character is mapped to a
183  // non-BMP character requiring two 2-byte units.
184  //
185  // Moreover, non-BMP characters in legacy multibyte encodings
186  // (e.g. EUC-JP, GB18030) take at least 2 bytes. The only exceptions are
187  // BOCU and SCSU, but we don't care about them.
188  size_t uchar_max_length = encoded.length() + 1;
189
190  SetUpErrorHandlerForToUChars(on_error, converter, &status);
191  scoped_ptr<char16[]> buffer(new char16[uchar_max_length]);
192  int actual_size = ucnv_toUChars(converter, buffer.get(),
193      static_cast<int>(uchar_max_length), encoded.data(),
194      static_cast<int>(encoded.length()), &status);
195  ucnv_close(converter);
196  if (!U_SUCCESS(status)) {
197    utf16->clear();  // Make sure the output is empty on error.
198    return false;
199  }
200
201  utf16->assign(buffer.get(), actual_size);
202  return true;
203}
204
205bool WideToCodepage(const std::wstring& wide,
206                    const char* codepage_name,
207                    OnStringConversionError::Type on_error,
208                    std::string* encoded) {
209#if defined(WCHAR_T_IS_UTF16)
210  return UTF16ToCodepage(wide, codepage_name, on_error, encoded);
211#elif defined(WCHAR_T_IS_UTF32)
212  encoded->clear();
213
214  UErrorCode status = U_ZERO_ERROR;
215  UConverter* converter = ucnv_open(codepage_name, &status);
216  if (!U_SUCCESS(status))
217    return false;
218
219  int utf16_len;
220  // When wchar_t is wider than UChar (16 bits), transform |wide| into a
221  // UChar* string.  Size the UChar* buffer to be large enough to hold twice
222  // as many UTF-16 code units (UChar's) as there are Unicode code points,
223  // in case each code points translates to a UTF-16 surrogate pair,
224  // and leave room for a NUL terminator.
225  std::vector<UChar> utf16(wide.length() * 2 + 1);
226  u_strFromUTF32(&utf16[0], utf16.size(), &utf16_len,
227                 reinterpret_cast<const UChar32*>(wide.c_str()),
228                 wide.length(), &status);
229  DCHECK(U_SUCCESS(status)) << "failed to convert wstring to UChar*";
230
231  return ConvertFromUTF16(converter, &utf16[0], utf16_len, on_error, encoded);
232#endif  // defined(WCHAR_T_IS_UTF32)
233}
234
235bool CodepageToWide(const std::string& encoded,
236                    const char* codepage_name,
237                    OnStringConversionError::Type on_error,
238                    std::wstring* wide) {
239#if defined(WCHAR_T_IS_UTF16)
240  return CodepageToUTF16(encoded, codepage_name, on_error, wide);
241#elif defined(WCHAR_T_IS_UTF32)
242  wide->clear();
243
244  UErrorCode status = U_ZERO_ERROR;
245  UConverter* converter = ucnv_open(codepage_name, &status);
246  if (!U_SUCCESS(status))
247    return false;
248
249  // The maximum length in 4 byte unit of UTF-32 output would be
250  // at most the same as the number of bytes in input. In the worst
251  // case of GB18030 (excluding escaped-based encodings like ISO-2022-JP),
252  // this can be 4 times larger than actually needed.
253  size_t wchar_max_length = encoded.length() + 1;
254
255  SetUpErrorHandlerForToUChars(on_error, converter, &status);
256  scoped_ptr<wchar_t[]> buffer(new wchar_t[wchar_max_length]);
257  int actual_size = ucnv_toAlgorithmic(utf32_platform_endian(), converter,
258      reinterpret_cast<char*>(buffer.get()),
259      static_cast<int>(wchar_max_length) * sizeof(wchar_t), encoded.data(),
260      static_cast<int>(encoded.length()), &status);
261  ucnv_close(converter);
262  if (!U_SUCCESS(status)) {
263    wide->clear();  // Make sure the output is empty on error.
264    return false;
265  }
266
267  // actual_size is # of bytes.
268  wide->assign(buffer.get(), actual_size / sizeof(wchar_t));
269  return true;
270#endif  // defined(WCHAR_T_IS_UTF32)
271}
272
273bool ConvertToUtf8AndNormalize(const std::string& text,
274                               const std::string& charset,
275                               std::string* result) {
276  result->clear();
277  string16 utf16;
278  if (!CodepageToUTF16(
279      text, charset.c_str(), OnStringConversionError::FAIL, &utf16))
280    return false;
281
282  UErrorCode status = U_ZERO_ERROR;
283  size_t max_length = utf16.length() + 1;
284  string16 normalized_utf16;
285  scoped_ptr<char16[]> buffer(new char16[max_length]);
286  int actual_length = unorm_normalize(
287      utf16.c_str(), utf16.length(), UNORM_NFC, 0,
288      buffer.get(), static_cast<int>(max_length), &status);
289  if (!U_SUCCESS(status))
290    return false;
291  normalized_utf16.assign(buffer.get(), actual_length);
292
293  return UTF16ToUTF8(normalized_utf16.data(),
294                     normalized_utf16.length(), result);
295}
296
297}  // namespace base
298