1589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// Copyright 2011 the V8 project authors. All rights reserved. 2589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// Redistribution and use in source and binary forms, with or without 3589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// modification, are permitted provided that the following conditions are 4589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// met: 5589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// 6589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// * Redistributions of source code must retain the above copyright 7589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// notice, this list of conditions and the following disclaimer. 8589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// * Redistributions in binary form must reproduce the above 9589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// copyright notice, this list of conditions and the following 10589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// disclaimer in the documentation and/or other materials provided 11589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// with the distribution. 12589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// * Neither the name of Google Inc. nor the names of its 13589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// contributors may be used to endorse or promote products derived 14589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// from this software without specific prior written permission. 15589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// 16589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 28589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch#include "v8.h" 29589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 30589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch#include "scanner-character-streams.h" 31589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 32589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch#include "handles.h" 33589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch#include "unicode-inl.h" 34589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 35589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdochnamespace v8 { 36589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdochnamespace internal { 37589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 38589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// ---------------------------------------------------------------------------- 393ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch// BufferedUtf16CharacterStreams 40589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 413ef787dbeca8a5fb1086949cda830dccee07bfbdBen MurdochBufferedUtf16CharacterStream::BufferedUtf16CharacterStream() 423ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch : Utf16CharacterStream(), 43589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch pushback_limit_(NULL) { 44589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Initialize buffer as being empty. First read will fill the buffer. 45589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch buffer_cursor_ = buffer_; 46589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch buffer_end_ = buffer_; 47589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 48589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 493ef787dbeca8a5fb1086949cda830dccee07bfbdBen MurdochBufferedUtf16CharacterStream::~BufferedUtf16CharacterStream() { } 50589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 513ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid BufferedUtf16CharacterStream::PushBack(uc32 character) { 52589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (character == kEndOfInput) { 53589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch pos_--; 54589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return; 55589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 56589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (pushback_limit_ == NULL && buffer_cursor_ > buffer_) { 57589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // buffer_ is writable, buffer_cursor_ is const pointer. 58589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch buffer_[--buffer_cursor_ - buffer_] = static_cast<uc16>(character); 59589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch pos_--; 60589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return; 61589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 62589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch SlowPushBack(static_cast<uc16>(character)); 63589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 64589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 65589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 663ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid BufferedUtf16CharacterStream::SlowPushBack(uc16 character) { 67589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // In pushback mode, the end of the buffer contains pushback, 68589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // and the start of the buffer (from buffer start to pushback_limit_) 69589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // contains valid data that comes just after the pushback. 70589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // We NULL the pushback_limit_ if pushing all the way back to the 71589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // start of the buffer. 72589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 73589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (pushback_limit_ == NULL) { 74589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Enter pushback mode. 75589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch pushback_limit_ = buffer_end_; 76589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch buffer_end_ = buffer_ + kBufferSize; 77589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch buffer_cursor_ = buffer_end_; 78589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 79589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Ensure that there is room for at least one pushback. 80589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ASSERT(buffer_cursor_ > buffer_); 81589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ASSERT(pos_ > 0); 82589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch buffer_[--buffer_cursor_ - buffer_] = character; 83589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (buffer_cursor_ == buffer_) { 84589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch pushback_limit_ = NULL; 85589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (buffer_cursor_ < pushback_limit_) { 86589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch pushback_limit_ = buffer_cursor_; 87589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 88589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch pos_--; 89589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 90589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 91589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 923ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochbool BufferedUtf16CharacterStream::ReadBlock() { 93589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch buffer_cursor_ = buffer_; 94589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (pushback_limit_ != NULL) { 95589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Leave pushback mode. 96589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch buffer_end_ = pushback_limit_; 97589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch pushback_limit_ = NULL; 98589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // If there were any valid characters left at the 99589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // start of the buffer, use those. 100589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (buffer_cursor_ < buffer_end_) return true; 101589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Otherwise read a new block. 102589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 103589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch unsigned length = FillBuffer(pos_, kBufferSize); 104589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch buffer_end_ = buffer_ + length; 105589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return length > 0; 106589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 107589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 108589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1093ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochunsigned BufferedUtf16CharacterStream::SlowSeekForward(unsigned delta) { 110589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Leave pushback mode (i.e., ignore that there might be valid data 111589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // in the buffer before the pushback_limit_ point). 112589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch pushback_limit_ = NULL; 113589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return BufferSeekForward(delta); 114589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 115589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 116589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// ---------------------------------------------------------------------------- 1173ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch// GenericStringUtf16CharacterStream 118589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 119589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1203ef787dbeca8a5fb1086949cda830dccee07bfbdBen MurdochGenericStringUtf16CharacterStream::GenericStringUtf16CharacterStream( 121589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Handle<String> data, 122589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch unsigned start_position, 123589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch unsigned end_position) 124589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch : string_(data), 125589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch length_(end_position) { 126589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ASSERT(end_position >= start_position); 127589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch buffer_cursor_ = buffer_; 128589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch buffer_end_ = buffer_; 129589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch pos_ = start_position; 130589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 131589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 132589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1333ef787dbeca8a5fb1086949cda830dccee07bfbdBen MurdochGenericStringUtf16CharacterStream::~GenericStringUtf16CharacterStream() { } 134589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 135589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1363ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochunsigned GenericStringUtf16CharacterStream::BufferSeekForward(unsigned delta) { 137589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch unsigned old_pos = pos_; 138589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch pos_ = Min(pos_ + delta, length_); 139589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ReadBlock(); 140589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return pos_ - old_pos; 141589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 142589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 143589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1443ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochunsigned GenericStringUtf16CharacterStream::FillBuffer(unsigned from_pos, 145589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch unsigned length) { 146589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (from_pos >= length_) return 0; 147589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (from_pos + length > length_) { 148589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch length = length_ - from_pos; 149589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 150589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch String::WriteToFlat<uc16>(*string_, buffer_, from_pos, from_pos + length); 151589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return length; 152589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 153589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 154589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 155589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// ---------------------------------------------------------------------------- 1563ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch// Utf8ToUtf16CharacterStream 1573ef787dbeca8a5fb1086949cda830dccee07bfbdBen MurdochUtf8ToUtf16CharacterStream::Utf8ToUtf16CharacterStream(const byte* data, 1583ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch unsigned length) 1593ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch : BufferedUtf16CharacterStream(), 160589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch raw_data_(data), 161589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch raw_data_length_(length), 162589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch raw_data_pos_(0), 163589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch raw_character_position_(0) { 164589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ReadBlock(); 165589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 166589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 167589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1683ef787dbeca8a5fb1086949cda830dccee07bfbdBen MurdochUtf8ToUtf16CharacterStream::~Utf8ToUtf16CharacterStream() { } 169589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 170589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1713ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochunsigned Utf8ToUtf16CharacterStream::BufferSeekForward(unsigned delta) { 172589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch unsigned old_pos = pos_; 173589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch unsigned target_pos = pos_ + delta; 174589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch SetRawPosition(target_pos); 175589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch pos_ = raw_character_position_; 176589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ReadBlock(); 177589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return pos_ - old_pos; 178589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 179589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 180589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1813ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochunsigned Utf8ToUtf16CharacterStream::FillBuffer(unsigned char_position, 1823ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch unsigned length) { 1833ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch static const unibrow::uchar kMaxUtf16Character = 0xffff; 184589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch SetRawPosition(char_position); 185589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (raw_character_position_ != char_position) { 186589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // char_position was not a valid position in the stream (hit the end 187589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // while spooling to it). 188589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return 0u; 189589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 190589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch unsigned i = 0; 1913ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch while (i < length - 1) { 192589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (raw_data_pos_ == raw_data_length_) break; 193589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch unibrow::uchar c = raw_data_[raw_data_pos_]; 194589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c <= unibrow::Utf8::kMaxOneByteChar) { 195589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch raw_data_pos_++; 196589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 197589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch c = unibrow::Utf8::CalculateValue(raw_data_ + raw_data_pos_, 198589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch raw_data_length_ - raw_data_pos_, 199589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch &raw_data_pos_); 200589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 2013ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (c > kMaxUtf16Character) { 2023ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch buffer_[i++] = unibrow::Utf16::LeadSurrogate(c); 2033ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch buffer_[i++] = unibrow::Utf16::TrailSurrogate(c); 2043ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } else { 2053ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch buffer_[i++] = static_cast<uc16>(c); 2063ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 207589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 208589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch raw_character_position_ = char_position + i; 209589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return i; 210589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 211589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 212589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 213589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdochstatic const byte kUtf8MultiByteMask = 0xC0; 214589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdochstatic const byte kUtf8MultiByteCharStart = 0xC0; 215589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdochstatic const byte kUtf8MultiByteCharFollower = 0x80; 216589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 217589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 218589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch#ifdef DEBUG 219589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdochstatic bool IsUtf8MultiCharacterStart(byte first_byte) { 220589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return (first_byte & kUtf8MultiByteMask) == kUtf8MultiByteCharStart; 221589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 222589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch#endif 223589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 224589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 225589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdochstatic bool IsUtf8MultiCharacterFollower(byte later_byte) { 226589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return (later_byte & kUtf8MultiByteMask) == kUtf8MultiByteCharFollower; 227589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 228589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 229589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 230589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// Move the cursor back to point at the preceding UTF-8 character start 231589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// in the buffer. 232589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdochstatic inline void Utf8CharacterBack(const byte* buffer, unsigned* cursor) { 233589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch byte character = buffer[--*cursor]; 234589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (character > unibrow::Utf8::kMaxOneByteChar) { 235589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ASSERT(IsUtf8MultiCharacterFollower(character)); 236589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Last byte of a multi-byte character encoding. Step backwards until 237589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // pointing to the first byte of the encoding, recognized by having the 238589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // top two bits set. 239589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (IsUtf8MultiCharacterFollower(buffer[--*cursor])) { } 240589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ASSERT(IsUtf8MultiCharacterStart(buffer[*cursor])); 241589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 242589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 243589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 244589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 245589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// Move the cursor forward to point at the next following UTF-8 character start 246589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// in the buffer. 247589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdochstatic inline void Utf8CharacterForward(const byte* buffer, unsigned* cursor) { 248589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch byte character = buffer[(*cursor)++]; 249589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (character > unibrow::Utf8::kMaxOneByteChar) { 250589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // First character of a multi-byte character encoding. 251589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // The number of most-significant one-bits determines the length of the 252589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // encoding: 253589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // 110..... - (0xCx, 0xDx) one additional byte (minimum). 254589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // 1110.... - (0xEx) two additional bytes. 255589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // 11110... - (0xFx) three additional bytes (maximum). 256589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ASSERT(IsUtf8MultiCharacterStart(character)); 257589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Additional bytes is: 258589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // 1 if value in range 0xC0 .. 0xDF. 259589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // 2 if value in range 0xE0 .. 0xEF. 260589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // 3 if value in range 0xF0 .. 0xF7. 261589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Encode that in a single value. 262589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch unsigned additional_bytes = 263589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ((0x3211u) >> (((character - 0xC0) >> 2) & 0xC)) & 0x03; 264589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch *cursor += additional_bytes; 265589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ASSERT(!IsUtf8MultiCharacterFollower(buffer[1 + additional_bytes])); 266589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 267589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 268589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 269589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 2703ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch// This can't set a raw position between two surrogate pairs, since there 2713ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch// is no position in the UTF8 stream that corresponds to that. This assumes 2723ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch// that the surrogate pair is correctly coded as a 4 byte UTF-8 sequence. If 2733ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch// it is illegally coded as two 3 byte sequences then there is no problem here. 2743ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid Utf8ToUtf16CharacterStream::SetRawPosition(unsigned target_position) { 275589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (raw_character_position_ > target_position) { 276589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Spool backwards in utf8 buffer. 277589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch do { 2783ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch int old_pos = raw_data_pos_; 279589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Utf8CharacterBack(raw_data_, &raw_data_pos_); 280589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch raw_character_position_--; 2813ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch ASSERT(old_pos - raw_data_pos_ <= 4); 2823ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Step back over both code units for surrogate pairs. 2833ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (old_pos - raw_data_pos_ == 4) raw_character_position_--; 284589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } while (raw_character_position_ > target_position); 2853ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // No surrogate pair splitting. 2863ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch ASSERT(raw_character_position_ == target_position); 287589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return; 288589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 289589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Spool forwards in the utf8 buffer. 290589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (raw_character_position_ < target_position) { 291589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (raw_data_pos_ == raw_data_length_) return; 2923ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch int old_pos = raw_data_pos_; 293589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Utf8CharacterForward(raw_data_, &raw_data_pos_); 294589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch raw_character_position_++; 2953ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch ASSERT(raw_data_pos_ - old_pos <= 4); 2963ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (raw_data_pos_ - old_pos == 4) raw_character_position_++; 297589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 2983ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // No surrogate pair splitting. 2993ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch ASSERT(raw_character_position_ == target_position); 300589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 301589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 302589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 303589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// ---------------------------------------------------------------------------- 3043ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch// ExternalTwoByteStringUtf16CharacterStream 305589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 3063ef787dbeca8a5fb1086949cda830dccee07bfbdBen MurdochExternalTwoByteStringUtf16CharacterStream:: 3073ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch ~ExternalTwoByteStringUtf16CharacterStream() { } 308589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 309589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 3103ef787dbeca8a5fb1086949cda830dccee07bfbdBen MurdochExternalTwoByteStringUtf16CharacterStream 3113ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch ::ExternalTwoByteStringUtf16CharacterStream( 312589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Handle<ExternalTwoByteString> data, 313589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int start_position, 314589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int end_position) 3153ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch : Utf16CharacterStream(), 316589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch source_(data), 317589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch raw_data_(data->GetTwoByteData(start_position)) { 318589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch buffer_cursor_ = raw_data_, 319589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch buffer_end_ = raw_data_ + (end_position - start_position); 320589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch pos_ = start_position; 321589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 322589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 323589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} } // namespace v8::internal 324