unicode-inl.h revision 3ef787dbeca8a5fb1086949cda830dccee07bfbd 
1// Copyright 2007-2010 the V8 project authors. All rights reserved.
2// Redistribution and use in source and binary forms, with or without
3// modification, are permitted provided that the following conditions are
4// met:
5//
6//     * Redistributions of source code must retain the above copyright
7//       notice, this list of conditions and the following disclaimer.
8//     * Redistributions in binary form must reproduce the above
9//       copyright notice, this list of conditions and the following
10//       disclaimer in the documentation and/or other materials provided
11//       with the distribution.
12//     * Neither the name of Google Inc. nor the names of its
13//       contributors may be used to endorse or promote products derived
14//       from this software without specific prior written permission.
15//
16// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28#ifndef V8_UNICODE_INL_H_
29#define V8_UNICODE_INL_H_
30
31#include "unicode.h"
32
33namespace unibrow {
34
35template <class T, int s> bool Predicate<T, s>::get(uchar code_point) {
36  CacheEntry entry = entries_[code_point & kMask];
37  if (entry.code_point_ == code_point) return entry.value_;
38  return CalculateValue(code_point);
39}
40
41template <class T, int s> bool Predicate<T, s>::CalculateValue(
42    uchar code_point) {
43  bool result = T::Is(code_point);
44  entries_[code_point & kMask] = CacheEntry(code_point, result);
45  return result;
46}
47
48template <class T, int s> int Mapping<T, s>::get(uchar c, uchar n,
49    uchar* result) {
50  CacheEntry entry = entries_[c & kMask];
51  if (entry.code_point_ == c) {
52    if (entry.offset_ == 0) {
53      return 0;
54    } else {
55      result[0] = c + entry.offset_;
56      return 1;
57    }
58  } else {
59    return CalculateValue(c, n, result);
60  }
61}
62
63template <class T, int s> int Mapping<T, s>::CalculateValue(uchar c, uchar n,
64    uchar* result) {
65  bool allow_caching = true;
66  int length = T::Convert(c, n, result, &allow_caching);
67  if (allow_caching) {
68    if (length == 1) {
69      entries_[c & kMask] = CacheEntry(c, result[0] - c);
70      return 1;
71    } else {
72      entries_[c & kMask] = CacheEntry(c, 0);
73      return 0;
74    }
75  } else {
76    return length;
77  }
78}
79
80
81unsigned Utf8::Encode(char* str, uchar c, int previous) {
82  static const int kMask = ~(1 << 6);
83  if (c <= kMaxOneByteChar) {
84    str[0] = c;
85    return 1;
86  } else if (c <= kMaxTwoByteChar) {
87    str[0] = 0xC0 | (c >> 6);
88    str[1] = 0x80 | (c & kMask);
89    return 2;
90  } else if (c <= kMaxThreeByteChar) {
91    if (Utf16::IsTrailSurrogate(c) &&
92        Utf16::IsLeadSurrogate(previous)) {
93      const int kUnmatchedSize = kSizeOfUnmatchedSurrogate;
94      return Encode(str - kUnmatchedSize,
95                    Utf16::CombineSurrogatePair(previous, c),
96                    Utf16::kNoPreviousCharacter) - kUnmatchedSize;
97    }
98    str[0] = 0xE0 | (c >> 12);
99    str[1] = 0x80 | ((c >> 6) & kMask);
100    str[2] = 0x80 | (c & kMask);
101    return 3;
102  } else {
103    str[0] = 0xF0 | (c >> 18);
104    str[1] = 0x80 | ((c >> 12) & kMask);
105    str[2] = 0x80 | ((c >> 6) & kMask);
106    str[3] = 0x80 | (c & kMask);
107    return 4;
108  }
109}
110
111
112uchar Utf8::ValueOf(const byte* bytes, unsigned length, unsigned* cursor) {
113  if (length <= 0) return kBadChar;
114  byte first = bytes[0];
115  // Characters between 0000 and 0007F are encoded as a single character
116  if (first <= kMaxOneByteChar) {
117    *cursor += 1;
118    return first;
119  }
120  return CalculateValue(bytes, length, cursor);
121}
122
123unsigned Utf8::Length(uchar c, int previous) {
124  if (c <= kMaxOneByteChar) {
125    return 1;
126  } else if (c <= kMaxTwoByteChar) {
127    return 2;
128  } else if (c <= kMaxThreeByteChar) {
129    if (Utf16::IsTrailSurrogate(c) &&
130        Utf16::IsLeadSurrogate(previous)) {
131      return kSizeOfUnmatchedSurrogate - kBytesSavedByCombiningSurrogates;
132    }
133    return 3;
134  } else {
135    return 4;
136  }
137}
138
139uchar CharacterStream::GetNext() {
140  uchar result = DecodeCharacter(buffer_, &cursor_);
141  if (remaining_ == 1) {
142    cursor_ = 0;
143    FillBuffer();
144  } else {
145    remaining_--;
146  }
147  return result;
148}
149
150#if __BYTE_ORDER == __LITTLE_ENDIAN
151#define IF_LITTLE(expr) expr
152#define IF_BIG(expr)    ((void) 0)
153#elif __BYTE_ORDER == __BIG_ENDIAN
154#define IF_LITTLE(expr) ((void) 0)
155#define IF_BIG(expr)    expr
156#else
157#warning Unknown byte ordering
158#endif
159
160bool CharacterStream::EncodeAsciiCharacter(uchar c, byte* buffer,
161    unsigned capacity, unsigned& offset) {
162  if (offset >= capacity) return false;
163  buffer[offset] = c;
164  offset += 1;
165  return true;
166}
167
168bool CharacterStream::EncodeNonAsciiCharacter(uchar c, byte* buffer,
169    unsigned capacity, unsigned& offset) {
170  unsigned aligned = (offset + 0x3) & ~0x3;
171  if ((aligned + sizeof(uchar)) > capacity)
172    return false;
173  if (offset == aligned) {
174    IF_LITTLE(*reinterpret_cast<uchar*>(buffer + aligned) = (c << 8) | 0x80);
175    IF_BIG(*reinterpret_cast<uchar*>(buffer + aligned) = c | (1 << 31));
176  } else {
177    buffer[offset] = 0x80;
178    IF_LITTLE(*reinterpret_cast<uchar*>(buffer + aligned) = c << 8);
179    IF_BIG(*reinterpret_cast<uchar*>(buffer + aligned) = c);
180  }
181  offset = aligned + sizeof(uchar);
182  return true;
183}
184
185bool CharacterStream::EncodeCharacter(uchar c, byte* buffer, unsigned capacity,
186    unsigned& offset) {
187  if (c <= Utf8::kMaxOneByteChar) {
188    return EncodeAsciiCharacter(c, buffer, capacity, offset);
189  } else {
190    return EncodeNonAsciiCharacter(c, buffer, capacity, offset);
191  }
192}
193
194uchar CharacterStream::DecodeCharacter(const byte* buffer, unsigned* offset) {
195  byte b = buffer[*offset];
196  if (b <= Utf8::kMaxOneByteChar) {
197    (*offset)++;
198    return b;
199  } else {
200    unsigned aligned = (*offset + 0x3) & ~0x3;
201    *offset = aligned + sizeof(uchar);
202    IF_LITTLE(return *reinterpret_cast<const uchar*>(buffer + aligned) >> 8);
203    IF_BIG(return *reinterpret_cast<const uchar*>(buffer + aligned) &
204                    ~(1 << 31));
205  }
206}
207
208#undef IF_LITTLE
209#undef IF_BIG
210
211template <class R, class I, unsigned s>
212void InputBuffer<R, I, s>::FillBuffer() {
213  buffer_ = R::ReadBlock(input_, util_buffer_, s, &remaining_, &offset_);
214}
215
216template <class R, class I, unsigned s>
217void InputBuffer<R, I, s>::Rewind() {
218  Reset(input_);
219}
220
221template <class R, class I, unsigned s>
222void InputBuffer<R, I, s>::Reset(unsigned position, I input) {
223  input_ = input;
224  remaining_ = 0;
225  cursor_ = 0;
226  offset_ = position;
227  buffer_ = R::ReadBlock(input_, util_buffer_, s, &remaining_, &offset_);
228}
229
230template <class R, class I, unsigned s>
231void InputBuffer<R, I, s>::Reset(I input) {
232  Reset(0, input);
233}
234
235template <class R, class I, unsigned s>
236void InputBuffer<R, I, s>::Seek(unsigned position) {
237  offset_ = position;
238  buffer_ = R::ReadBlock(input_, util_buffer_, s, &remaining_, &offset_);
239}
240
241template <unsigned s>
242Utf8InputBuffer<s>::Utf8InputBuffer(const char* data, unsigned length)
243    : InputBuffer<Utf8, Buffer<const char*>, s>(Buffer<const char*>(data,
244                                                                    length)) {
245}
246
247}  // namespace unibrow
248
249#endif  // V8_UNICODE_INL_H_
250