18b112d2025046f85ef7f6be087c6129c872ebad2Ben Murdoch// Copyright 2011 the V8 project authors. All rights reserved. 2b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// Use of this source code is governed by a BSD-style license that can be 3b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// found in the LICENSE file. 4a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 5589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// Features shared by parsing and pre-parsing scanners. 6589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 7a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block#ifndef V8_SCANNER_H_ 8a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block#define V8_SCANNER_H_ 9a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 10b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/allocation.h" 11b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/base/logging.h" 12b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/char-predicates.h" 13b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/globals.h" 14b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/hashmap.h" 15b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/list.h" 16b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/token.h" 17b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/unicode-inl.h" 18b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/utils.h" 19a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 20a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Blocknamespace v8 { 21a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Blocknamespace internal { 22a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 233ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 24b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochclass AstRawString; 25b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochclass AstValueFactory; 26b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochclass ParserRecorder; 273ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 283ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 29589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// Returns the value (0 .. 15) of a hexadecimal character c. 30589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// If c is not a legal hexadecimal character, returns a value < 0. 31589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdochinline int HexValue(uc32 c) { 32589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch c -= '0'; 33589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (static_cast<unsigned>(c) <= 9) return c; 34589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch c = (c | 0x20) - ('a' - '0'); // detect 0x11..0x16 and 0x31..0x36. 35589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (static_cast<unsigned>(c) <= 5) return c + 10; 36589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return -1; 37589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 38589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 39589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 40589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// --------------------------------------------------------------------- 413ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch// Buffered stream of UTF-16 code units, using an internal UTF-16 buffer. 423ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch// A code unit is a 16 bit value representing either a 16 bit code point 433ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch// or one part of a surrogate pair that make a single 21 bit code point. 44589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 453ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochclass Utf16CharacterStream { 46a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block public: 473ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Utf16CharacterStream() : pos_(0) { } 483ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch virtual ~Utf16CharacterStream() { } 49589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 503ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Returns and advances past the next UTF-16 code unit in the input 513ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // stream. If there are no more code units, it returns a negative 52589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // value. 53589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch inline uc32 Advance() { 54589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (buffer_cursor_ < buffer_end_ || ReadBlock()) { 55589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch pos_++; 56589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return static_cast<uc32>(*(buffer_cursor_++)); 57589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 58589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Note: currently the following increment is necessary to avoid a 59589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // parser problem! The scanner treats the final kEndOfInput as 603ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // a code unit with a position, and does math relative to that 61589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // position. 62589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch pos_++; 63589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 64589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return kEndOfInput; 65589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 66b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 673ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Return the current position in the code unit stream. 68589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Starts at zero. 69589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch inline unsigned pos() const { return pos_; } 70589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 713ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Skips forward past the next code_unit_count UTF-16 code units 72589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // in the input, or until the end of input if that comes sooner. 733ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Returns the number of code units actually skipped. If less 743ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // than code_unit_count, 753ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch inline unsigned SeekForward(unsigned code_unit_count) { 76589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch unsigned buffered_chars = 77589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch static_cast<unsigned>(buffer_end_ - buffer_cursor_); 783ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (code_unit_count <= buffered_chars) { 793ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch buffer_cursor_ += code_unit_count; 803ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch pos_ += code_unit_count; 813ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch return code_unit_count; 82589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 833ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch return SlowSeekForward(code_unit_count); 84589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 85589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 863ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Pushes back the most recently read UTF-16 code unit (or negative 87589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // value if at end of input), i.e., the value returned by the most recent 88589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // call to Advance. 89589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Must not be used right after calling SeekForward. 903ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch virtual void PushBack(int32_t code_unit) = 0; 91b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 92b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch protected: 93589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch static const uc32 kEndOfInput = -1; 94589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 953ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Ensures that the buffer_cursor_ points to the code_unit at 96589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // position pos_ of the input, if possible. If the position 97589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // is at or after the end of the input, return false. If there 983ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // are more code_units available, return true. 99589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch virtual bool ReadBlock() = 0; 1003ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch virtual unsigned SlowSeekForward(unsigned code_unit_count) = 0; 101589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 102b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch const uint16_t* buffer_cursor_; 103b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch const uint16_t* buffer_end_; 104589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch unsigned pos_; 105589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch}; 106589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 107b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 108589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// --------------------------------------------------------------------- 109589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// Caching predicates used by scanners. 110b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 111b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochclass UnicodeCache { 112589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch public: 113589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch UnicodeCache() {} 114b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch typedef unibrow::Utf8Decoder<512> Utf8Decoder; 115589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 116589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch StaticResource<Utf8Decoder>* utf8_decoder() { 117589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return &utf8_decoder_; 118589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 119589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 120589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch bool IsIdentifierStart(unibrow::uchar c) { return kIsIdentifierStart.get(c); } 121589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch bool IsIdentifierPart(unibrow::uchar c) { return kIsIdentifierPart.get(c); } 122589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch bool IsLineTerminator(unibrow::uchar c) { return kIsLineTerminator.get(c); } 123589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch bool IsWhiteSpace(unibrow::uchar c) { return kIsWhiteSpace.get(c); } 124b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool IsWhiteSpaceOrLineTerminator(unibrow::uchar c) { 125b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return kIsWhiteSpaceOrLineTerminator.get(c); 126b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 127b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 128589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch private: 129589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch unibrow::Predicate<IdentifierStart, 128> kIsIdentifierStart; 130589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch unibrow::Predicate<IdentifierPart, 128> kIsIdentifierPart; 131589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch unibrow::Predicate<unibrow::LineTerminator, 128> kIsLineTerminator; 132b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch unibrow::Predicate<WhiteSpace, 128> kIsWhiteSpace; 133b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch unibrow::Predicate<WhiteSpaceOrLineTerminator, 128> 134b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch kIsWhiteSpaceOrLineTerminator; 135589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch StaticResource<Utf8Decoder> utf8_decoder_; 136b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 137589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch DISALLOW_COPY_AND_ASSIGN(UnicodeCache); 138a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block}; 139a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 140a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 141b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// --------------------------------------------------------------------- 142b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// DuplicateFinder discovers duplicate symbols. 143b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 144b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochclass DuplicateFinder { 145b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch public: 146b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch explicit DuplicateFinder(UnicodeCache* constants) 147b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch : unicode_constants_(constants), 148b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch backing_store_(16), 149b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch map_(&Match) { } 150b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 151b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int AddOneByteSymbol(Vector<const uint8_t> key, int value); 152b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int AddTwoByteSymbol(Vector<const uint16_t> key, int value); 153b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Add a a number literal by converting it (if necessary) 154b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // to the string that ToString(ToNumber(literal)) would generate. 155b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // and then adding that string with AddOneByteSymbol. 156b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // This string is the actual value used as key in an object literal, 157b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // and the one that must be different from the other keys. 158b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int AddNumber(Vector<const uint8_t> key, int value); 159b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 160b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch private: 161b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int AddSymbol(Vector<const uint8_t> key, bool is_one_byte, int value); 162b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Backs up the key and its length in the backing store. 163b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // The backup is stored with a base 127 encoding of the 164b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // length (plus a bit saying whether the string is one byte), 165b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // followed by the bytes of the key. 166b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch uint8_t* BackupKey(Vector<const uint8_t> key, bool is_one_byte); 167b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 168b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Compare two encoded keys (both pointing into the backing store) 169b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // for having the same base-127 encoded lengths and representation. 170b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // and then having the same 'length' bytes following. 171b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch static bool Match(void* first, void* second); 172b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Creates a hash from a sequence of bytes. 173b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch static uint32_t Hash(Vector<const uint8_t> key, bool is_one_byte); 174b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Checks whether a string containing a JS number is its canonical 175b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // form. 176b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch static bool IsNumberCanonical(Vector<const uint8_t> key); 177b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 178b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Size of buffer. Sufficient for using it to call DoubleToCString in 179b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // from conversions.h. 180b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch static const int kBufferSize = 100; 181b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 182b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch UnicodeCache* unicode_constants_; 183b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Backing store used to store strings used as hashmap keys. 184b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch SequenceCollector<unsigned char> backing_store_; 185b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch HashMap map_; 186b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Buffer used for string->number->canonical string conversions. 187b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch char number_buffer_[kBufferSize]; 188b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch}; 189b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 190b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 191589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// ---------------------------------------------------------------------------- 192589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// LiteralBuffer - Collector of chars of literals. 193589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 194589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdochclass LiteralBuffer { 195a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block public: 196b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch LiteralBuffer() : is_one_byte_(true), position_(0), backing_store_() { } 197b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 198589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ~LiteralBuffer() { 199589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (backing_store_.length() > 0) { 200589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch backing_store_.Dispose(); 201589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 202589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 203589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 2043ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch INLINE(void AddChar(uint32_t code_unit)) { 205589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (position_ >= backing_store_.length()) ExpandBuffer(); 206b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (is_one_byte_) { 207b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (code_unit <= unibrow::Latin1::kMaxChar) { 2083ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch backing_store_[position_] = static_cast<byte>(code_unit); 209b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch position_ += kOneByteSize; 210589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return; 211589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 212b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch ConvertToTwoByte(); 213589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 214b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(code_unit < 0x10000u); 215b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch *reinterpret_cast<uint16_t*>(&backing_store_[position_]) = code_unit; 216589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch position_ += kUC16Size; 217589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 218589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 219b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool is_one_byte() const { return is_one_byte_; } 220b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 221b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool is_contextual_keyword(Vector<const char> keyword) const { 222b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return is_one_byte() && keyword.length() == position_ && 223b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch (memcmp(keyword.start(), backing_store_.start(), position_) == 0); 224b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 225589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 226b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Vector<const uint16_t> two_byte_literal() const { 227b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(!is_one_byte_); 228b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK((position_ & 0x1) == 0); 229b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return Vector<const uint16_t>( 230b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch reinterpret_cast<const uint16_t*>(backing_store_.start()), 231589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch position_ >> 1); 232589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 233589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 234b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Vector<const uint8_t> one_byte_literal() const { 235b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(is_one_byte_); 236b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return Vector<const uint8_t>( 237b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch reinterpret_cast<const uint8_t*>(backing_store_.start()), 238589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch position_); 239589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 240b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 241b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int length() const { 242b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return is_one_byte_ ? position_ : (position_ >> 1); 243589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 244589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 245589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch void Reset() { 246589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch position_ = 0; 247b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch is_one_byte_ = true; 248589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 249589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 250b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Handle<String> Internalize(Isolate* isolate) const; 251b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 252589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch private: 253589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch static const int kInitialCapacity = 16; 254589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch static const int kGrowthFactory = 4; 255589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch static const int kMinConversionSlack = 256; 256589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch static const int kMaxGrowth = 1 * MB; 257589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch inline int NewCapacity(int min_capacity) { 258589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int capacity = Max(min_capacity, backing_store_.length()); 259589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int new_capacity = Min(capacity * kGrowthFactory, capacity + kMaxGrowth); 260589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return new_capacity; 261589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 262589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 263589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch void ExpandBuffer() { 264589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Vector<byte> new_store = Vector<byte>::New(NewCapacity(kInitialCapacity)); 265b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch MemCopy(new_store.start(), backing_store_.start(), position_); 266589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch backing_store_.Dispose(); 267589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch backing_store_ = new_store; 268589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 269589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 270b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch void ConvertToTwoByte() { 271b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(is_one_byte_); 272589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Vector<byte> new_store; 273589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int new_content_size = position_ * kUC16Size; 274589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (new_content_size >= backing_store_.length()) { 2753ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Ensure room for all currently read code units as UC16 as well 2763ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // as the code unit about to be stored. 277589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch new_store = Vector<byte>::New(NewCapacity(new_content_size)); 278589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 279589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch new_store = backing_store_; 280589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 281b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch uint8_t* src = backing_store_.start(); 282b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch uint16_t* dst = reinterpret_cast<uint16_t*>(new_store.start()); 283589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch for (int i = position_ - 1; i >= 0; i--) { 284589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch dst[i] = src[i]; 285589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 286589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (new_store.start() != backing_store_.start()) { 287589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch backing_store_.Dispose(); 288589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch backing_store_ = new_store; 289589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 290589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch position_ = new_content_size; 291b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch is_one_byte_ = false; 292589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 293589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 294b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool is_one_byte_; 295589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int position_; 296589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Vector<byte> backing_store_; 297589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 298589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch DISALLOW_COPY_AND_ASSIGN(LiteralBuffer); 299a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block}; 300a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 301a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 302589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// ---------------------------------------------------------------------------- 3033ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch// JavaScript Scanner. 304589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 305589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdochclass Scanner { 306a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block public: 3073ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Scoped helper for literal recording. Automatically drops the literal 3083ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // if aborting the scanning before it's complete. 309589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch class LiteralScope { 310589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch public: 3113ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch explicit LiteralScope(Scanner* self) 3123ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch : scanner_(self), complete_(false) { 3133ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch scanner_->StartLiteral(); 3143ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 3153ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch ~LiteralScope() { 3163ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch if (!complete_) scanner_->DropLiteral(); 3173ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 3183ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch void Complete() { 3193ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch scanner_->TerminateLiteral(); 3203ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch complete_ = true; 3213ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 322589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 323589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch private: 324589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Scanner* scanner_; 325589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch bool complete_; 326589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch }; 327589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 3283ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Representation of an interval of source positions. 329589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch struct Location { 330589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Location(int b, int e) : beg_pos(b), end_pos(e) { } 331589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Location() : beg_pos(0), end_pos(0) { } 332589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 333589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch bool IsValid() const { 334589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return beg_pos >= 0 && end_pos >= beg_pos; 335589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 336589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 337589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch static Location invalid() { return Location(-1, -1); } 338589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 339589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int beg_pos; 340589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int end_pos; 341589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch }; 342589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 3433ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // -1 is outside of the range of any real source code. 3443ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch static const int kNoOctalLocation = -1; 3453ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 3463ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch explicit Scanner(UnicodeCache* scanner_contants); 3473ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 3483ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch void Initialize(Utf16CharacterStream* source); 3493ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 3503ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Returns the next token and advances input. 3513ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Token::Value Next(); 3523ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Returns the current token again. 3533ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Token::Value current_token() { return current_.token; } 354589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Returns the location information for the current token 3553ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // (the token last returned by Next()). 356589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Location location() const { return current_.location; } 357b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 358b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Similar functions for the upcoming token. 359b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 360b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // One token look-ahead (past the token returned by Next()). 361b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Token::Value peek() const { return next_.token; } 362b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 363b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Location peek_location() const { return next_.location; } 364589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 365589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch bool literal_contains_escapes() const { 366589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Location location = current_.location; 367589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int source_length = (location.end_pos - location.beg_pos); 368589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (current_.token == Token::STRING) { 369589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Subtract delimiters. 370589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch source_length -= 2; 371589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 372589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return current_.literal_chars->length() != source_length; 373589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 374b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool is_literal_contextual_keyword(Vector<const char> keyword) { 375b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK_NOT_NULL(current_.literal_chars); 376b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return current_.literal_chars->is_contextual_keyword(keyword); 377592a9fc1d8ea420377a2e7efd0600e20b058be2bBen Murdoch } 378b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool is_next_contextual_keyword(Vector<const char> keyword) { 379b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK_NOT_NULL(next_.literal_chars); 380b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return next_.literal_chars->is_contextual_keyword(keyword); 381589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 382589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 383b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch const AstRawString* CurrentSymbol(AstValueFactory* ast_value_factory); 384b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch const AstRawString* NextSymbol(AstValueFactory* ast_value_factory); 385589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 386b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch double DoubleValue(); 387b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool UnescapedLiteralMatches(const char* data, int length) { 388b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (is_literal_one_byte() && 389b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch literal_length() == length && 390b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch !literal_contains_escapes()) { 391b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch const char* token = 392b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch reinterpret_cast<const char*>(literal_one_byte_string().start()); 393b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return !strncmp(token, data, length); 394b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 395b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return false; 396b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 397b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch void IsGetOrSet(bool* is_get, bool* is_set) { 398b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (is_literal_one_byte() && 399b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch literal_length() == 3 && 400b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch !literal_contains_escapes()) { 401b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch const char* token = 402b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch reinterpret_cast<const char*>(literal_one_byte_string().start()); 403b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch *is_get = strncmp(token, "get", 3) == 0; 404b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch *is_set = !*is_get && strncmp(token, "set", 3) == 0; 405b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 406b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 407b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 408b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int FindNumber(DuplicateFinder* finder, int value); 409b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int FindSymbol(DuplicateFinder* finder, int value); 410b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 411b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch UnicodeCache* unicode_cache() { return unicode_cache_; } 4123ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 4133ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Returns the location of the last seen octal literal. 4143ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Location octal_position() const { return octal_pos_; } 4153ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch void clear_octal_position() { octal_pos_ = Location::invalid(); } 4163ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 4173ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Seek forward to the given position. This operation does not 4183ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // work in general, for instance when there are pushed back 4193ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // characters, but works for seeking forward until simple delimiter 4203ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // tokens, which is what it is used for. 4213ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch void SeekForward(int pos); 4223ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 4233ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch bool HarmonyScoping() const { 4243ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch return harmony_scoping_; 4253ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 4263ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch void SetHarmonyScoping(bool scoping) { 4273ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch harmony_scoping_ = scoping; 4283ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 4293ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch bool HarmonyModules() const { 4303ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch return harmony_modules_; 4313ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 4323ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch void SetHarmonyModules(bool modules) { 4333ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch harmony_modules_ = modules; 4343ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 435b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool HarmonyNumericLiterals() const { 436b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return harmony_numeric_literals_; 437b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 438b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch void SetHarmonyNumericLiterals(bool numeric_literals) { 439b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch harmony_numeric_literals_ = numeric_literals; 440b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 441b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool HarmonyClasses() const { 442b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return harmony_classes_; 443b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 444b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch void SetHarmonyClasses(bool classes) { 445b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch harmony_classes_ = classes; 446b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 4473ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 4483ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Returns true if there was a line terminator before the peek'ed token, 4493ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // possibly inside a multi-line comment. 4503ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch bool HasAnyLineTerminatorBeforeNext() const { 4513ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch return has_line_terminator_before_next_ || 4523ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch has_multiline_comment_before_next_; 4533ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 4543ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 4553ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Scans the input as a regular expression pattern, previous 4563ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // character(s) must be /(=). Returns true if a pattern is scanned. 4573ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch bool ScanRegExpPattern(bool seen_equal); 4583ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Returns true if regexp flags are scanned (always since flags can 4593ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // be empty). 4603ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch bool ScanRegExpFlags(); 4613ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 462b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch const LiteralBuffer* source_url() const { return &source_url_; } 463b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch const LiteralBuffer* source_mapping_url() const { 464b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return &source_mapping_url_; 465b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 466b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 467b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool IdentifierIsFutureStrictReserved(const AstRawString* string) const; 4683ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 4693ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch private: 470589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // The current and look-ahead token. 471589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch struct TokenDesc { 472589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::Value token; 473589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Location location; 474589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch LiteralBuffer* literal_chars; 475589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch }; 476589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 477b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch static const int kCharacterLookaheadBufferSize = 1; 478b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 479b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Scans octal escape sequence. Also accepts "\0" decimal escape sequence. 480b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch uc32 ScanOctalEscape(uc32 c, int length); 481b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 482589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Call this after setting source_ to the input. 483589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch void Init() { 484589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Set c0_ (one character ahead) 485589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch STATIC_ASSERT(kCharacterLookaheadBufferSize == 1); 486589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 487589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Initialize current_ to not refer to a literal. 488589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch current_.literal_chars = NULL; 489589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 490589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 491589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Literal buffer support 492589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch inline void StartLiteral() { 493589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch LiteralBuffer* free_buffer = (current_.literal_chars == &literal_buffer1_) ? 494589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch &literal_buffer2_ : &literal_buffer1_; 495589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch free_buffer->Reset(); 496589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.literal_chars = free_buffer; 497589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 498589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 4993ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch INLINE(void AddLiteralChar(uc32 c)) { 500b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK_NOT_NULL(next_.literal_chars); 501589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.literal_chars->AddChar(c); 502589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 503589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 504589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Complete scanning of a literal. 505589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch inline void TerminateLiteral() { 506589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Does nothing in the current implementation. 507589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 508589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 509589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Stops scanning of a literal and drop the collected characters, 510589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // e.g., due to an encountered error. 511589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch inline void DropLiteral() { 512589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.literal_chars = NULL; 513589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 514589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 515589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch inline void AddLiteralCharAdvance() { 516589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralChar(c0_); 517589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 518589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 519589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 520589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Low-level scanning support. 521589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch void Advance() { c0_ = source_->Advance(); } 522589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch void PushBack(uc32 ch) { 523589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch source_->PushBack(c0_); 524589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch c0_ = ch; 525589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 526589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 527589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch inline Token::Value Select(Token::Value tok) { 528589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 529589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return tok; 530589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 531589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 532589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch inline Token::Value Select(uc32 next, Token::Value then, Token::Value else_) { 533589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 534589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == next) { 535589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 536589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return then; 537589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 538589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return else_; 539589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 540589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 541589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 542b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Returns the literal string, if any, for the current token (the 543b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // token last returned by Next()). The string is 0-terminated. 544b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Literal strings are collected for identifiers, strings, and 545b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // numbers. 546b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // These functions only give the correct result if the literal 547b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // was scanned between calls to StartLiteral() and TerminateLiteral(). 548b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Vector<const uint8_t> literal_one_byte_string() { 549b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK_NOT_NULL(current_.literal_chars); 550b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return current_.literal_chars->one_byte_literal(); 551b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 552b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Vector<const uint16_t> literal_two_byte_string() { 553b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK_NOT_NULL(current_.literal_chars); 554b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return current_.literal_chars->two_byte_literal(); 555b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 556b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool is_literal_one_byte() { 557b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK_NOT_NULL(current_.literal_chars); 558b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return current_.literal_chars->is_one_byte(); 559b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 560b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int literal_length() const { 561b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK_NOT_NULL(current_.literal_chars); 562b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return current_.literal_chars->length(); 563b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 564b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Returns the literal string for the next token (the token that 565b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // would be returned if Next() were called). 566b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Vector<const uint8_t> next_literal_one_byte_string() { 567b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK_NOT_NULL(next_.literal_chars); 568b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return next_.literal_chars->one_byte_literal(); 569b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 570b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Vector<const uint16_t> next_literal_two_byte_string() { 571b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK_NOT_NULL(next_.literal_chars); 572b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return next_.literal_chars->two_byte_literal(); 573b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 574b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool is_next_literal_one_byte() { 575b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK_NOT_NULL(next_.literal_chars); 576b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return next_.literal_chars->is_one_byte(); 577b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 578b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int next_literal_length() const { 579b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK_NOT_NULL(next_.literal_chars); 580b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return next_.literal_chars->length(); 581b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 582b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 583589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 ScanHexNumber(int expected_length); 584589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 5853ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Scans a single JavaScript token. 5863ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch void Scan(); 58785b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch 588589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch bool SkipWhiteSpace(); 589589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::Value SkipSingleLineComment(); 590b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Token::Value SkipSourceURLComment(); 591b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch void TryToParseSourceURLComment(); 592589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::Value SkipMultiLineComment(); 5933ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Scans a possible HTML comment -- begins with '<!'. 5943ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Token::Value ScanHtmlComment(); 595589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 596589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch void ScanDecimalDigits(); 597589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::Value ScanNumber(bool seen_period); 598589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::Value ScanIdentifierOrKeyword(); 599589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::Value ScanIdentifierSuffix(LiteralScope* literal); 600589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 601589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::Value ScanString(); 602589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 603b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Scans an escape-sequence which is part of a string and adds the 604b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // decoded character to the current literal. Returns true if a pattern 605b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // is scanned. 606b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool ScanEscape(); 607b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Decodes a Unicode escape-sequence which is part of an identifier. 608589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // If the escape sequence cannot be decoded the result is kBadChar. 609589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 ScanIdentifierUnicodeEscape(); 610b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Scans a Unicode escape-sequence and adds its characters, 611589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // uninterpreted, to the current literal. Used for parsing RegExp 612589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // flags. 613589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch bool ScanLiteralUnicodeEscape(); 614589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 6153ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Return the current source position. 6163ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch int source_pos() { 6173ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch return source_->pos() - kCharacterLookaheadBufferSize; 6183ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch } 6193ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 6203ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch UnicodeCache* unicode_cache_; 6213ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 6223ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Buffers collecting literal strings, numbers, etc. 6233ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch LiteralBuffer literal_buffer1_; 6243ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch LiteralBuffer literal_buffer2_; 6253ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 626b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Values parsed from magic comments. 627b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch LiteralBuffer source_url_; 628b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch LiteralBuffer source_mapping_url_; 629b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 6303ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch TokenDesc current_; // desc for current token (as returned by Next()) 6313ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch TokenDesc next_; // desc for next token (one token look-ahead) 6323ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 6333ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Input stream. Must be initialized to an Utf16CharacterStream. 6343ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Utf16CharacterStream* source_; 6353ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 6363ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 637589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Start position of the octal literal last scanned. 638589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Location octal_pos_; 639589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 6403ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // One Unicode character look-ahead; c0_ < 0 at the end of the input. 6413ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch uc32 c0_; 6423ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch 643589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Whether there is a line terminator whitespace character after 644589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // the current token, and before the next. Does not count newlines 645589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // inside multiline comments. 646589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch bool has_line_terminator_before_next_; 647589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Whether there is a multi-line comment that contains a 648589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // line-terminator after the current token, and before the next. 649589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch bool has_multiline_comment_before_next_; 6503ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Whether we scan 'let' as a keyword for harmony block-scoped let bindings. 6513ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch bool harmony_scoping_; 6523ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Whether we scan 'module', 'import', 'export' as keywords. 6533ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch bool harmony_modules_; 654b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Whether we scan 0o777 and 0b111 as numbers. 655b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool harmony_numeric_literals_; 656b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Whether we scan 'class', 'extends', 'static' and 'super' as keywords. 657b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool harmony_classes_; 658b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch}; 659b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 660a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block} } // namespace v8::internal 661a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 662a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block#endif // V8_SCANNER_H_ 663