file_util_icu.cc revision c407dc5cd9bdc5668497f21b26b09d988ab439de 
1// Copyright (c) 2009 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5// File utilities that use the ICU library go in this file.
6
7#include "base/i18n/file_util_icu.h"
8
9#include "base/file_path.h"
10#include "base/logging.h"
11#include "base/scoped_ptr.h"
12#include "base/singleton.h"
13#include "base/string_util.h"
14#include "base/utf_string_conversions.h"
15#include "base/sys_string_conversions.h"
16#include "build/build_config.h"
17#include "unicode/coll.h"
18#include "unicode/uniset.h"
19
20namespace {
21
22class IllegalCharacters {
23 public:
24  bool contains(UChar32 ucs4) {
25    return !!set->contains(ucs4);
26  }
27
28  bool containsNone(const string16 &s) {
29    return !!set->containsNone(icu::UnicodeString(s.c_str(), s.size()));
30  }
31
32 private:
33  friend class Singleton<IllegalCharacters>;
34  friend struct DefaultSingletonTraits<IllegalCharacters>;
35
36  IllegalCharacters();
37  ~IllegalCharacters() { }
38
39  scoped_ptr<icu::UnicodeSet> set;
40
41  DISALLOW_COPY_AND_ASSIGN(IllegalCharacters);
42};
43
44IllegalCharacters::IllegalCharacters() {
45  UErrorCode status = U_ZERO_ERROR;
46  // Control characters, formatting characters, non-characters, and
47  // some printable ASCII characters regarded as dangerous ('"*/:<>?\\').
48  // See  http://blogs.msdn.com/michkap/archive/2006/11/03/941420.aspx
49  // and http://msdn2.microsoft.com/en-us/library/Aa365247.aspx
50  // TODO(jungshik): Revisit the set. ZWJ and ZWNJ are excluded because they
51  // are legitimate in Arabic and some S/SE Asian scripts. However, when used
52  // elsewhere, they can be confusing/problematic.
53  // Also, consider wrapping the set with our Singleton class to create and
54  // freeze it only once. Note that there's a trade-off between memory and
55  // speed.
56#if defined(WCHAR_T_IS_UTF16)
57  set.reset(new icu::UnicodeSet(icu::UnicodeString(
58      L"[[\"*/:<>?\\\\|][:Cc:][:Cf:] - [\u200c\u200d]]"), status));
59#else
60  set.reset(new icu::UnicodeSet(UNICODE_STRING_SIMPLE(
61      "[[\"*/:<>?\\\\|][:Cc:][:Cf:] - [\\u200c\\u200d]]").unescape(),
62      status));
63#endif
64  DCHECK(U_SUCCESS(status));
65  // Add non-characters. If this becomes a performance bottleneck by
66  // any chance, do not add these to |set| and change IsFilenameLegal()
67  // to check |ucs4 & 0xFFFEu == 0xFFFEu|, in addiition to calling
68  // containsNone().
69  set->add(0xFDD0, 0xFDEF);
70  for (int i = 0; i <= 0x10; ++i) {
71    int plane_base = 0x10000 * i;
72    set->add(plane_base + 0xFFFE, plane_base + 0xFFFF);
73  }
74  set->freeze();
75}
76
77class LocaleAwareComparator {
78 public:
79  LocaleAwareComparator() {
80    UErrorCode error_code = U_ZERO_ERROR;
81    // Use the default collator. The default locale should have been properly
82    // set by the time this constructor is called.
83    collator_.reset(icu::Collator::createInstance(error_code));
84    DCHECK(U_SUCCESS(error_code));
85    // Make it case-sensitive.
86    collator_->setStrength(icu::Collator::TERTIARY);
87    // Note: We do not set UCOL_NORMALIZATION_MODE attribute. In other words, we
88    // do not pay performance penalty to guarantee sort order correctness for
89    // non-FCD (http://unicode.org/notes/tn5/#FCD) file names. This should be a
90    // reasonable tradeoff because such file names should be rare and the sort
91    // order doesn't change much anyway.
92  }
93
94  // Note: A similar function is available in l10n_util.
95  // We cannot use it because base should not depend on l10n_util.
96  // TODO(yuzo): Move some of l10n_util to base.
97  int Compare(const string16& a, const string16& b) {
98    // We are not sure if Collator::compare is thread-safe.
99    // Use an AutoLock just in case.
100    AutoLock auto_lock(lock_);
101
102    UErrorCode error_code = U_ZERO_ERROR;
103    UCollationResult result = collator_->compare(
104        static_cast<const UChar*>(a.c_str()),
105        static_cast<int>(a.length()),
106        static_cast<const UChar*>(b.c_str()),
107        static_cast<int>(b.length()),
108        error_code);
109    DCHECK(U_SUCCESS(error_code));
110    return result;
111  }
112
113 private:
114  scoped_ptr<icu::Collator> collator_;
115  Lock lock_;
116  friend struct DefaultSingletonTraits<LocaleAwareComparator>;
117
118  DISALLOW_COPY_AND_ASSIGN(LocaleAwareComparator);
119};
120
121}  // namespace
122
123namespace file_util {
124
125bool IsFilenameLegal(const string16& file_name) {
126  return Singleton<IllegalCharacters>()->containsNone(file_name);
127}
128
129void ReplaceIllegalCharactersInPath(FilePath::StringType* file_name,
130                                    char replace_char) {
131  DCHECK(file_name);
132
133  DCHECK(!(Singleton<IllegalCharacters>()->contains(replace_char)));
134
135  // Remove leading and trailing whitespace.
136  TrimWhitespace(*file_name, TRIM_ALL, file_name);
137
138  IllegalCharacters* illegal = Singleton<IllegalCharacters>::get();
139  int cursor = 0;  // The ICU macros expect an int.
140  while (cursor < static_cast<int>(file_name->size())) {
141    int char_begin = cursor;
142    uint32 code_point;
143#if defined(OS_MACOSX)
144    // Mac uses UTF-8 encoding for filenames.
145    U8_NEXT(file_name->data(), cursor, static_cast<int>(file_name->length()),
146            code_point);
147#elif defined(OS_WIN)
148    // Windows uses UTF-16 encoding for filenames.
149    U16_NEXT(file_name->data(), cursor, static_cast<int>(file_name->length()),
150             code_point);
151#elif defined(OS_POSIX)
152    // Linux doesn't actually define an encoding. It basically allows anything
153    // except for a few special ASCII characters.
154    unsigned char cur_char = static_cast<unsigned char>((*file_name)[cursor++]);
155    if (cur_char >= 0x80)
156      continue;
157    code_point = cur_char;
158#else
159    NOTREACHED();
160#endif
161
162    if (illegal->contains(code_point)) {
163      file_name->replace(char_begin, cursor - char_begin, 1, replace_char);
164      // We just made the potentially multi-byte/word char into one that only
165      // takes one byte/word, so need to adjust the cursor to point to the next
166      // character again.
167      cursor = char_begin + 1;
168    }
169  }
170}
171
172bool LocaleAwareCompareFilenames(const FilePath& a, const FilePath& b) {
173#if defined(OS_WIN)
174  return Singleton<LocaleAwareComparator>()->Compare(a.value().c_str(),
175                                                     b.value().c_str()) < 0;
176
177#elif defined(OS_POSIX)
178  // On linux, the file system encoding is not defined. We assume
179  // SysNativeMBToWide takes care of it.
180  //
181  // ICU's collator can take strings in OS native encoding. But we convert the
182  // strings to UTF-16 ourselves to ensure conversion consistency.
183  // TODO(yuzo): Perhaps we should define SysNativeMBToUTF16?
184  return Singleton<LocaleAwareComparator>()->Compare(
185      WideToUTF16(base::SysNativeMBToWide(a.value().c_str())),
186      WideToUTF16(base::SysNativeMBToWide(b.value().c_str()))) < 0;
187#else
188  #error Not implemented on your system
189#endif
190}
191
192}  // namespace
193