scanner.cc revision 85b71799222b55eb5dd74ea26efe0c64ab655c8c
18b112d2025046f85ef7f6be087c6129c872ebad2Ben Murdoch// Copyright 2011 the V8 project authors. All rights reserved. 2a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// Redistribution and use in source and binary forms, with or without 3a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// modification, are permitted provided that the following conditions are 4a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// met: 5a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// 6a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// * Redistributions of source code must retain the above copyright 7a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// notice, this list of conditions and the following disclaimer. 8a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// * Redistributions in binary form must reproduce the above 9a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// copyright notice, this list of conditions and the following 10a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// disclaimer in the documentation and/or other materials provided 11a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// with the distribution. 12a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// * Neither the name of Google Inc. nor the names of its 13a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// contributors may be used to endorse or promote products derived 14a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// from this software without specific prior written permission. 15a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// 16a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 28589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// Features shared by parsing and pre-parsing scanners. 29a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 30a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block#include "scanner.h" 31589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 32589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch#include "../include/v8stdint.h" 33589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch#include "char-predicates-inl.h" 34a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 35a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Blocknamespace v8 { 36a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Blocknamespace internal { 37a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 38a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// ---------------------------------------------------------------------------- 3985b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch// Scanner::LiteralScope 40a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 4185b71799222b55eb5dd74ea26efe0c64ab655c8cBen MurdochScanner::LiteralScope::LiteralScope(Scanner* self) 4285b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch : scanner_(self), complete_(false) { 4385b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch self->StartLiteral(); 4485b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch} 45a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 46a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 4785b71799222b55eb5dd74ea26efe0c64ab655c8cBen MurdochScanner::LiteralScope::~LiteralScope() { 4885b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch if (!complete_) scanner_->DropLiteral(); 4985b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch} 5085b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch 5185b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch 5285b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdochvoid Scanner::LiteralScope::Complete() { 5385b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch scanner_->TerminateLiteral(); 5485b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch complete_ = true; 55589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 56b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 5785b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch// ---------------------------------------------------------------------------- 5885b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch// Scanner 5985b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch 6085b71799222b55eb5dd74ea26efe0c64ab655c8cBen MurdochScanner::Scanner(UnicodeCache* unicode_cache) 6185b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch : unicode_cache_(unicode_cache) { } 6285b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch 63589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 64589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdochuc32 Scanner::ScanHexNumber(int expected_length) { 65589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ASSERT(expected_length <= 4); // prevent overflow 66589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 67589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 digits[4] = { 0, 0, 0, 0 }; 68589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 x = 0; 69589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch for (int i = 0; i < expected_length; i++) { 70589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch digits[i] = c0_; 71589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int d = HexValue(c0_); 72589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (d < 0) { 73589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // According to ECMA-262, 3rd, 7.8.4, page 18, these hex escapes 74589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // should be illegal, but other JS VMs just return the 75589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // non-escaped version of the original character. 76589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 77589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Push back digits that we have advanced past. 78589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch for (int j = i-1; j >= 0; j--) { 79589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch PushBack(digits[j]); 80589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 81589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return -1; 82589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 83589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch x = x * 16 + d; 84589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 85b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch } 86589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 87589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return x; 88a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block} 89a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 90a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 9185b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch 9285b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch// ---------------------------------------------------------------------------- 9385b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch// JavaScriptScanner 9485b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch 9585b71799222b55eb5dd74ea26efe0c64ab655c8cBen MurdochJavaScriptScanner::JavaScriptScanner(UnicodeCache* scanner_contants) 9685b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch : Scanner(scanner_contants), 9785b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch octal_pos_(Location::invalid()), 9885b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch harmony_block_scoping_(false) { } 9985b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch 10085b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch 10185b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdochvoid JavaScriptScanner::Initialize(UC16CharacterStream* source) { 10285b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch source_ = source; 10385b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch // Need to capture identifiers in order to recognize "get" and "set" 10485b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch // in object literals. 10585b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch Init(); 10685b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch // Skip initial whitespace allowing HTML comment ends just like 10785b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch // after a newline and scan first token. 10885b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch has_line_terminator_before_next_ = true; 10985b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch SkipWhiteSpace(); 11085b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch Scan(); 11185b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch} 11285b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch 11385b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch 114589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// Ensure that tokens can be stored in a byte. 115589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben MurdochSTATIC_ASSERT(Token::NUM_TOKENS <= 0x100); 116589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 117589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// Table of one-character tokens, by character (0x00..0x7f only). 118589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdochstatic const byte one_char_tokens[] = { 119589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 120589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 121589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 122589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 123589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 124589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 125589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 126589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 127589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 128589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 129589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 130589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 131589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 132589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 133589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 134589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 135589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 136589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 137589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 138589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 139589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 140589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 141589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 142589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 143589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 144589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 145589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 146589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 147589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 148589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 149589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 150589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 151589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 152589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 153589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 154589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 155589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 156589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 157589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 158589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 159589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::LPAREN, // 0x28 160589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::RPAREN, // 0x29 161589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 162589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 163589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::COMMA, // 0x2c 164589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 165589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 166589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 167589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 168589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 169589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 170589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 171589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 172589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 173589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 174589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 175589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 176589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 177589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::COLON, // 0x3a 178589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::SEMICOLON, // 0x3b 179589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 180589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 181589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 182589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::CONDITIONAL, // 0x3f 183589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 184589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 185589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 186589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 187589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 188589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 189589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 190589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 191589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 192589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 193589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 194589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 195589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 196589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 197589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 198589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 199589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 200589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 201589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 202589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 203589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 204589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 205589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 206589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 207589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 208589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 209589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 210589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::LBRACK, // 0x5b 211589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 212589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::RBRACK, // 0x5d 213589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 214589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 215589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 216589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 217589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 218589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 219589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 220589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 221589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 222589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 223589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 224589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 225589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 226589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 227589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 228589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 229589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 230589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 231589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 232589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 233589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 234589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 235589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 236589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 237589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 238589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 239589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 240589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 241589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 242589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::LBRACE, // 0x7b 243589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 244589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::RBRACE, // 0x7d 245589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::BIT_NOT, // 0x7e 246589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL 247589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch}; 248589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 249589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 25085b71799222b55eb5dd74ea26efe0c64ab655c8cBen MurdochToken::Value JavaScriptScanner::Next() { 251589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch current_ = next_; 252589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch has_line_terminator_before_next_ = false; 253589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch has_multiline_comment_before_next_ = false; 254589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (static_cast<unsigned>(c0_) <= 0x7f) { 255589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::Value token = static_cast<Token::Value>(one_char_tokens[c0_]); 256589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (token != Token::ILLEGAL) { 257589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int pos = source_pos(); 258589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.token = token; 259589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.location.beg_pos = pos; 260589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.location.end_pos = pos + 1; 261589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 262589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return current_.token; 263589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 264a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block } 265589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Scan(); 266589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return current_.token; 267b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 268b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 269b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 270589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdochstatic inline bool IsByteOrderMark(uc32 c) { 271589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // The Unicode value U+FFFE is guaranteed never to be assigned as a 272589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Unicode character; this implies that in a Unicode context the 273589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // 0xFF, 0xFE byte pattern can only be interpreted as the U+FEFF 274589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // character expressed in little-endian byte order (since it could 275589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // not be a U+FFFE character expressed in big-endian byte 276589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // order). Nevertheless, we check for it to be compatible with 277589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Spidermonkey. 278589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return c == 0xFEFF || c == 0xFFFE; 279b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 280b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 281589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 28285b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdochbool JavaScriptScanner::SkipWhiteSpace() { 283589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int start_position = source_pos(); 284589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 285589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (true) { 286589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // We treat byte-order marks (BOMs) as whitespace for better 287589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // compatibility with Spidermonkey and other JavaScript engines. 288589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (unicode_cache_->IsWhiteSpace(c0_) || IsByteOrderMark(c0_)) { 289589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // IsWhiteSpace() includes line terminators! 290589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (unicode_cache_->IsLineTerminator(c0_)) { 291589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Ignore line terminators, but remember them. This is necessary 292589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // for automatic semicolon insertion. 293589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch has_line_terminator_before_next_ = true; 294589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 295589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 296589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 297589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 298589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // If there is an HTML comment end '-->' at the beginning of a 299589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // line (with only whitespace in front of it), we treat the rest 300589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // of the line as a comment. This is in line with the way 301589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // SpiderMonkey handles it. 302589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '-' && has_line_terminator_before_next_) { 303589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 304589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '-') { 305589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 306589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '>') { 307589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Treat the rest of the line as a comment. 308589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch SkipSingleLineComment(); 309589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Continue skipping white space after the comment. 310589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch continue; 311589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 312589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch PushBack('-'); // undo Advance() 313589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 314589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch PushBack('-'); // undo Advance() 315589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 316589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Return whether or not we skipped any characters. 317589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return source_pos() != start_position; 318589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 319b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 320b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 321b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 32285b71799222b55eb5dd74ea26efe0c64ab655c8cBen MurdochToken::Value JavaScriptScanner::SkipSingleLineComment() { 323589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 324b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 325589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // The line terminator at the end of the line is not considered 326589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // to be part of the single-line comment; it is recognized 327589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // separately by the lexical grammar and becomes part of the 328589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // stream of input elements for the syntactic grammar (see 329589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // ECMA-262, section 7.4). 330589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (c0_ >= 0 && !unicode_cache_->IsLineTerminator(c0_)) { 331589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 332589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 333b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 334589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::WHITESPACE; 335a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block} 336a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 337a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 33885b71799222b55eb5dd74ea26efe0c64ab655c8cBen MurdochToken::Value JavaScriptScanner::SkipMultiLineComment() { 339589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ASSERT(c0_ == '*'); 340589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 341589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 342589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (c0_ >= 0) { 343589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 ch = c0_; 344589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 345589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (unicode_cache_->IsLineTerminator(ch)) { 346589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Following ECMA-262, section 7.4, a comment containing 347589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // a newline will make the comment count as a line-terminator. 348589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch has_multiline_comment_before_next_ = true; 349589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 350589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // If we have reached the end of the multi-line comment, we 351589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // consume the '/' and insert a whitespace. This way all 352589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // multi-line comments are treated as whitespace. 353589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (ch == '*' && c0_ == '/') { 354589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch c0_ = ' '; 355589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::WHITESPACE; 356589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 357b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch } 358589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 359589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Unterminated multi-line comment. 360589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::ILLEGAL; 361b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 362b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 363b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 36485b71799222b55eb5dd74ea26efe0c64ab655c8cBen MurdochToken::Value JavaScriptScanner::ScanHtmlComment() { 365589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Check for <!-- comments. 366589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ASSERT(c0_ == '!'); 367589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 368589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '-') { 369589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 370589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '-') return SkipSingleLineComment(); 371589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch PushBack('-'); // undo Advance() 372589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 373589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch PushBack('!'); // undo Advance() 374589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ASSERT(c0_ == '!'); 375589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::LT; 376b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 377b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 378b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 37985b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdochvoid JavaScriptScanner::Scan() { 380589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.literal_chars = NULL; 381589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::Value token; 382589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch do { 383589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Remember the position of the next token 384589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.location.beg_pos = source_pos(); 385589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 386589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch switch (c0_) { 387589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case ' ': 388589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '\t': 389589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 390589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::WHITESPACE; 391589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 392589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 393589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '\n': 394589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 395589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch has_line_terminator_before_next_ = true; 396589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::WHITESPACE; 397589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 398589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 399589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '"': case '\'': 400589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = ScanString(); 401589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 402589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 403589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '<': 404589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // < <= << <<= <!-- 405589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 406589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '=') { 407589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::LTE); 408589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ == '<') { 409589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select('=', Token::ASSIGN_SHL, Token::SHL); 410589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ == '!') { 411589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = ScanHtmlComment(); 412589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 413589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::LT; 414589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 415589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 416589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 417589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '>': 418589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // > >= >> >>= >>> >>>= 419589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 420589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '=') { 421589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::GTE); 422589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ == '>') { 423589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // >> >>= >>> >>>= 424589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 425589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '=') { 426589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::ASSIGN_SAR); 427589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ == '>') { 428589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select('=', Token::ASSIGN_SHR, Token::SHR); 429589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 430589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::SAR; 431589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 432589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 433589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::GT; 434589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 435589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 436589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 437589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '=': 438589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // = == === 439589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 440589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '=') { 441589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select('=', Token::EQ_STRICT, Token::EQ); 442589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 443589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::ASSIGN; 444589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 445589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 446589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 447589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '!': 448589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // ! != !== 449589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 450589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '=') { 451589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select('=', Token::NE_STRICT, Token::NE); 452589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 453589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::NOT; 454589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 455589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 456589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 457589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '+': 458589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // + ++ += 459589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 460589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '+') { 461589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::INC); 462589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ == '=') { 463589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::ASSIGN_ADD); 464589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 465589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::ADD; 466589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 467589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 468589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 469589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '-': 470589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // - -- --> -= 471589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 472589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '-') { 473589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 474589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '>' && has_line_terminator_before_next_) { 475589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // For compatibility with SpiderMonkey, we skip lines that 476589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // start with an HTML comment end '-->'. 477589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = SkipSingleLineComment(); 478589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 479589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::DEC; 480589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 481589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ == '=') { 482589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::ASSIGN_SUB); 483589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 484589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::SUB; 485589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 486589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 487589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 488589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '*': 489589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // * *= 490589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select('=', Token::ASSIGN_MUL, Token::MUL); 491589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 492b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 493589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '%': 494589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // % %= 495589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select('=', Token::ASSIGN_MOD, Token::MOD); 496589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 497b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 498589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '/': 499589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // / // /* /= 500589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 501589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '/') { 502589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = SkipSingleLineComment(); 503589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ == '*') { 504589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = SkipMultiLineComment(); 505589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ == '=') { 506589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::ASSIGN_DIV); 507589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 508589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::DIV; 509589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 510589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 511589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 512589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '&': 513589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // & && &= 514589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 515589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '&') { 516589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::AND); 517589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ == '=') { 518589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::ASSIGN_BIT_AND); 519589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 520589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::BIT_AND; 521589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 522589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 523589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 524589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '|': 525589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // | || |= 526589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 527589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '|') { 528589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::OR); 529589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ == '=') { 530589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::ASSIGN_BIT_OR); 531589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 532589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::BIT_OR; 533589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 534589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 535589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 536589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '^': 537589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // ^ ^= 538589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select('=', Token::ASSIGN_BIT_XOR, Token::BIT_XOR); 539589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 540589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 541589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '.': 542589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // . Number 543589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 544589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (IsDecimalDigit(c0_)) { 545589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = ScanNumber(true); 546589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 547589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::PERIOD; 548589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 549589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 550589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 551589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case ':': 552589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::COLON); 553589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 554589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 555589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case ';': 556589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::SEMICOLON); 557589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 558589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 559589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case ',': 560589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::COMMA); 561589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 562589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 563589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '(': 564589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::LPAREN); 565589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 566589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 567589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case ')': 568589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::RPAREN); 569589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 570589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 571589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '[': 572589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::LBRACK); 573589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 574589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 575589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case ']': 576589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::RBRACK); 577589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 578589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 579589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '{': 580589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::LBRACE); 581589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 582589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 583589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '}': 584589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::RBRACE); 585589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 586589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 587589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '?': 588589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::CONDITIONAL); 589589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 590589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 591589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '~': 592589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::BIT_NOT); 593589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 594589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 595589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch default: 596589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (unicode_cache_->IsIdentifierStart(c0_)) { 597589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = ScanIdentifierOrKeyword(); 598589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (IsDecimalDigit(c0_)) { 599589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = ScanNumber(false); 600589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (SkipWhiteSpace()) { 601589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::WHITESPACE; 602589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ < 0) { 603589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::EOS; 604589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 605589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::ILLEGAL); 606589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 607589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 608589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 609589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 610589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Continue scanning for tokens as long as we're just skipping 611589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // whitespace. 612589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } while (token == Token::WHITESPACE); 613589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 614589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.location.end_pos = source_pos(); 615589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.token = token; 616b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 617b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 618b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 61985b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdochvoid JavaScriptScanner::SeekForward(int pos) { 620589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // After this call, we will have the token at the given position as 621589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // the "next" token. The "current" token will be invalid. 622589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (pos == next_.location.beg_pos) return; 623589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int current_pos = source_pos(); 624589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ASSERT_EQ(next_.location.end_pos, current_pos); 625589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Positions inside the lookahead token aren't supported. 626589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ASSERT(pos >= current_pos); 627589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (pos != current_pos) { 628589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch source_->SeekForward(pos - source_->pos()); 629589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 630589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // This function is only called to seek to the location 631589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // of the end of a function (at the "}" token). It doesn't matter 632589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // whether there was a line terminator in the part we skip. 633589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch has_line_terminator_before_next_ = false; 634589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch has_multiline_comment_before_next_ = false; 635b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch } 636589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Scan(); 637589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 638589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 639589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 64085b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdochvoid JavaScriptScanner::ScanEscape() { 641589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 c = c0_; 642589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 643589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 644589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Skip escaped newlines. 645589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (unicode_cache_->IsLineTerminator(c)) { 646589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Allow CR+LF newlines in multiline string literals. 647589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (IsCarriageReturn(c) && IsLineFeed(c0_)) Advance(); 648589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Allow LF+CR newlines in multiline string literals. 649589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (IsLineFeed(c) && IsCarriageReturn(c0_)) Advance(); 650589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return; 651589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 652589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 653589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch switch (c) { 654589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '\'': // fall through 655589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '"' : // fall through 656589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '\\': break; 657589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case 'b' : c = '\b'; break; 658589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case 'f' : c = '\f'; break; 659589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case 'n' : c = '\n'; break; 660589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case 'r' : c = '\r'; break; 661589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case 't' : c = '\t'; break; 662589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case 'u' : { 663589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch c = ScanHexNumber(4); 664589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c < 0) c = 'u'; 665589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 666589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 667589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case 'v' : c = '\v'; break; 668589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case 'x' : { 669589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch c = ScanHexNumber(2); 670589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c < 0) c = 'x'; 671589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 672b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch } 673589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '0' : // fall through 674589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '1' : // fall through 675589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '2' : // fall through 676589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '3' : // fall through 677589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '4' : // fall through 678589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '5' : // fall through 679589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '6' : // fall through 680589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '7' : c = ScanOctalEscape(c, 2); break; 681b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch } 682589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 683589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // According to ECMA-262, 3rd, 7.8.4 (p 18ff) these 684589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // should be illegal, but they are commonly handled 685589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // as non-escaped characters by JS VMs. 686589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralChar(c); 687b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 688b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 689b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 690589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// Octal escapes of the forms '\0xx' and '\xxx' are not a part of 691589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// ECMA-262. Other JS VMs support them. 69285b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdochuc32 JavaScriptScanner::ScanOctalEscape(uc32 c, int length) { 693589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 x = c - '0'; 694589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int i = 0; 695589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch for (; i < length; i++) { 696589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int d = c0_ - '0'; 697589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (d < 0 || d > 7) break; 698589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int nx = x * 8 + d; 699589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (nx >= 256) break; 700589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch x = nx; 701589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 702589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 703589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Anything except '\0' is an octal escape sequence, illegal in strict mode. 704589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Remember the position of octal escape sequences so that an error 705589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // can be reported later (in strict mode). 706589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // We don't report the error immediately, because the octal escape can 707589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // occur before the "use strict" directive. 708589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c != '0' || i > 0) { 709589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch octal_pos_ = Location(source_pos() - i - 1, source_pos() - 1); 710589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 711589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return x; 712589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 713589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 714589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 71585b71799222b55eb5dd74ea26efe0c64ab655c8cBen MurdochToken::Value JavaScriptScanner::ScanString() { 716589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 quote = c0_; 717589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); // consume quote 718b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 719589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch LiteralScope literal(this); 720589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (c0_ != quote && c0_ >= 0 721589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch && !unicode_cache_->IsLineTerminator(c0_)) { 722589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 c = c0_; 723589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 724589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c == '\\') { 725589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ < 0) return Token::ILLEGAL; 726589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ScanEscape(); 727589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 728589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralChar(c); 729589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 730589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 731589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ != quote) return Token::ILLEGAL; 732589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch literal.Complete(); 733b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 734589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); // consume quote 735589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::STRING; 736b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 737b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 738b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 73985b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdochvoid JavaScriptScanner::ScanDecimalDigits() { 740589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (IsDecimalDigit(c0_)) 741589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 742b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 743b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 744b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 74585b71799222b55eb5dd74ea26efe0c64ab655c8cBen MurdochToken::Value JavaScriptScanner::ScanNumber(bool seen_period) { 746589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ASSERT(IsDecimalDigit(c0_)); // the first digit of the number or the fraction 747589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 748589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch enum { DECIMAL, HEX, OCTAL } kind = DECIMAL; 749589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 750589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch LiteralScope literal(this); 751589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (seen_period) { 752589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // we have already seen a decimal point of the float 753589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralChar('.'); 754589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ScanDecimalDigits(); // we know we have at least one digit 755589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 756589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 757589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // if the first character is '0' we must check for octals and hex 758589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '0') { 759589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int start_pos = source_pos(); // For reporting octal positions. 760589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 761589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 762589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // either 0, 0exxx, 0Exxx, 0.xxx, an octal number, or a hex number 763589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == 'x' || c0_ == 'X') { 764589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // hex number 765589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch kind = HEX; 766589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 767589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (!IsHexDigit(c0_)) { 768589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // we must have at least one hex digit after 'x'/'X' 769589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::ILLEGAL; 770589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 771589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (IsHexDigit(c0_)) { 772589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 773589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 774589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if ('0' <= c0_ && c0_ <= '7') { 775589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // (possible) octal number 776589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch kind = OCTAL; 777589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (true) { 778589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '8' || c0_ == '9') { 779589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch kind = DECIMAL; 780589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 781589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 782589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ < '0' || '7' < c0_) { 783589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Octal literal finished. 784589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch octal_pos_ = Location(start_pos, source_pos()); 785589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 786589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 787589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 788589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 789589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 790589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 791589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 792589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Parse decimal digits and allow trailing fractional part. 793589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (kind == DECIMAL) { 794589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ScanDecimalDigits(); // optional 795589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '.') { 796589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 797589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ScanDecimalDigits(); // optional 798589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 799589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 800b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch } 801589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 802589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // scan exponent, if any 803589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == 'e' || c0_ == 'E') { 804589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ASSERT(kind != HEX); // 'e'/'E' must be scanned as part of the hex number 805589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (kind == OCTAL) return Token::ILLEGAL; // no exponent for octals allowed 806589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // scan exponent 807589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 808589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '+' || c0_ == '-') 809589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 810589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (!IsDecimalDigit(c0_)) { 811589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // we must have at least one decimal digit after 'e'/'E' 812589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::ILLEGAL; 813589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 814589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ScanDecimalDigits(); 815589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 816589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 817589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // The source character immediately following a numeric literal must 818589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // not be an identifier start or a decimal digit; see ECMA-262 819589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // section 7.8.3, page 17 (note that we read only one decimal digit 820589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // if the value is 0). 821589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (IsDecimalDigit(c0_) || unicode_cache_->IsIdentifierStart(c0_)) 822589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::ILLEGAL; 823589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 824589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch literal.Complete(); 825589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 826589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::NUMBER; 827b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 828b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 829b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 83085b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdochuc32 JavaScriptScanner::ScanIdentifierUnicodeEscape() { 831589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 832589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ != 'u') return -1; 833589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 834589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 result = ScanHexNumber(4); 835589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (result < 0) PushBack('u'); 836589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return result; 837589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 838589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 839589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 840589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// ---------------------------------------------------------------------------- 841589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// Keyword Matcher 842589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 843589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch#define KEYWORDS(KEYWORD_GROUP, KEYWORD) \ 844589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD_GROUP('b') \ 845589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("break", Token::BREAK) \ 846589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD_GROUP('c') \ 847589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("case", Token::CASE) \ 848589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("catch", Token::CATCH) \ 849589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("class", Token::FUTURE_RESERVED_WORD) \ 850589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("const", Token::CONST) \ 851589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("continue", Token::CONTINUE) \ 852589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD_GROUP('d') \ 853589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("debugger", Token::DEBUGGER) \ 854589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("default", Token::DEFAULT) \ 855589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("delete", Token::DELETE) \ 856589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("do", Token::DO) \ 857589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD_GROUP('e') \ 858589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("else", Token::ELSE) \ 859589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("enum", Token::FUTURE_RESERVED_WORD) \ 86085b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch KEYWORD("export", Token::FUTURE_RESERVED_WORD) \ 861589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("extends", Token::FUTURE_RESERVED_WORD) \ 862589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD_GROUP('f') \ 863589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("false", Token::FALSE_LITERAL) \ 864589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("finally", Token::FINALLY) \ 865589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("for", Token::FOR) \ 866589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("function", Token::FUNCTION) \ 867589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD_GROUP('i') \ 868589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("if", Token::IF) \ 869589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("implements", Token::FUTURE_STRICT_RESERVED_WORD) \ 87085b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch KEYWORD("import", Token::FUTURE_RESERVED_WORD) \ 871589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("in", Token::IN) \ 872589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("instanceof", Token::INSTANCEOF) \ 873589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("interface", Token::FUTURE_STRICT_RESERVED_WORD) \ 874589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD_GROUP('l') \ 87585b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch KEYWORD("let", harmony_block_scoping \ 876589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ? Token::LET : Token::FUTURE_STRICT_RESERVED_WORD) \ 877589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD_GROUP('n') \ 878589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("new", Token::NEW) \ 879589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("null", Token::NULL_LITERAL) \ 880589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD_GROUP('p') \ 881589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("package", Token::FUTURE_STRICT_RESERVED_WORD) \ 882589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("private", Token::FUTURE_STRICT_RESERVED_WORD) \ 883589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("protected", Token::FUTURE_STRICT_RESERVED_WORD) \ 884589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("public", Token::FUTURE_STRICT_RESERVED_WORD) \ 885589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD_GROUP('r') \ 886589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("return", Token::RETURN) \ 887589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD_GROUP('s') \ 888589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("static", Token::FUTURE_STRICT_RESERVED_WORD) \ 889589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("super", Token::FUTURE_RESERVED_WORD) \ 890589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("switch", Token::SWITCH) \ 891589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD_GROUP('t') \ 892589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("this", Token::THIS) \ 893589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("throw", Token::THROW) \ 894589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("true", Token::TRUE_LITERAL) \ 895589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("try", Token::TRY) \ 896589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("typeof", Token::TYPEOF) \ 897589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD_GROUP('v') \ 898589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("var", Token::VAR) \ 899589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("void", Token::VOID) \ 900589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD_GROUP('w') \ 901589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("while", Token::WHILE) \ 902589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("with", Token::WITH) \ 903589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD_GROUP('y') \ 904589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORD("yield", Token::FUTURE_STRICT_RESERVED_WORD) 905589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 906589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 907589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdochstatic Token::Value KeywordOrIdentifierToken(const char* input, 908589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int input_length, 90985b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch bool harmony_block_scoping) { 910589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ASSERT(input_length >= 1); 911589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch const int kMinLength = 2; 912589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch const int kMaxLength = 10; 913589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (input_length < kMinLength || input_length > kMaxLength) { 914589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::IDENTIFIER; 915b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch } 916589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch switch (input[0]) { 917589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch default: 918589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch#define KEYWORD_GROUP_CASE(ch) \ 919589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; \ 920589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case ch: 921589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch#define KEYWORD(keyword, token) \ 922589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch { \ 923589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch /* 'keyword' is a char array, so sizeof(keyword) is */ \ 924589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch /* strlen(keyword) plus 1 for the NUL char. */ \ 925589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch const int keyword_length = sizeof(keyword) - 1; \ 926589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch STATIC_ASSERT(keyword_length >= kMinLength); \ 927589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch STATIC_ASSERT(keyword_length <= kMaxLength); \ 928589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (input_length == keyword_length && \ 929589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch input[1] == keyword[1] && \ 930589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch (keyword_length <= 2 || input[2] == keyword[2]) && \ 931589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch (keyword_length <= 3 || input[3] == keyword[3]) && \ 932589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch (keyword_length <= 4 || input[4] == keyword[4]) && \ 933589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch (keyword_length <= 5 || input[5] == keyword[5]) && \ 934589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch (keyword_length <= 6 || input[6] == keyword[6]) && \ 935589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch (keyword_length <= 7 || input[7] == keyword[7]) && \ 936589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch (keyword_length <= 8 || input[8] == keyword[8]) && \ 937589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch (keyword_length <= 9 || input[9] == keyword[9])) { \ 938589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return token; \ 939589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } \ 940589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 941589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORDS(KEYWORD_GROUP_CASE, KEYWORD) 942589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 943589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::IDENTIFIER; 944b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 945b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 946b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 94785b71799222b55eb5dd74ea26efe0c64ab655c8cBen MurdochToken::Value JavaScriptScanner::ScanIdentifierOrKeyword() { 948589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ASSERT(unicode_cache_->IsIdentifierStart(c0_)); 949589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch LiteralScope literal(this); 950589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Scan identifier start character. 951589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '\\') { 952589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 c = ScanIdentifierUnicodeEscape(); 953589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Only allow legal identifier start characters. 954589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c < 0 || 955589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch c == '\\' || // No recursive escapes. 956589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch !unicode_cache_->IsIdentifierStart(c)) { 957589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::ILLEGAL; 958589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 959589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralChar(c); 960589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return ScanIdentifierSuffix(&literal); 961b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch } 962589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 963589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 first_char = c0_; 964589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 965589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralChar(first_char); 966589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 967589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Scan the rest of the identifier characters. 968589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (unicode_cache_->IsIdentifierPart(c0_)) { 969589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ != '\\') { 970589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 next_char = c0_; 971589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 972589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralChar(next_char); 973589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch continue; 974589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 975589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Fallthrough if no longer able to complete keyword. 976589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return ScanIdentifierSuffix(&literal); 977b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch } 978589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 979589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch literal.Complete(); 980589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 981589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (next_.literal_chars->is_ascii()) { 982589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Vector<const char> chars = next_.literal_chars->ascii_literal(); 983589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return KeywordOrIdentifierToken(chars.start(), 984589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch chars.length(), 98585b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch harmony_block_scoping_); 986589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 987589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 988589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::IDENTIFIER; 989b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 990b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 991b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 99285b71799222b55eb5dd74ea26efe0c64ab655c8cBen MurdochToken::Value JavaScriptScanner::ScanIdentifierSuffix(LiteralScope* literal) { 993589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Scan the rest of the identifier characters. 994589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (unicode_cache_->IsIdentifierPart(c0_)) { 995589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '\\') { 996589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 c = ScanIdentifierUnicodeEscape(); 997589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Only allow legal identifier part characters. 998589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c < 0 || 999589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch c == '\\' || 1000589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch !unicode_cache_->IsIdentifierPart(c)) { 1001589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::ILLEGAL; 1002589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1003589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralChar(c); 1004589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 1005589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralChar(c0_); 1006589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 1007589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1008589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1009589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch literal->Complete(); 1010589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1011589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::IDENTIFIER; 1012a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block} 1013a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 1014a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 101585b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdochbool JavaScriptScanner::ScanRegExpPattern(bool seen_equal) { 1016589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Scan: ('/' | '/=') RegularExpressionBody '/' RegularExpressionFlags 1017589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch bool in_character_class = false; 101880d68eab642096c1a48b6474d6ec33064b0ad1f5Kristian Monsen 1019589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Previous token is either '/' or '/=', in the second case, the 1020589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // pattern starts at =. 1021589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.location.beg_pos = source_pos() - (seen_equal ? 2 : 1); 1022589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.location.end_pos = source_pos() - (seen_equal ? 1 : 0); 1023589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1024589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Scan regular expression body: According to ECMA-262, 3rd, 7.8.5, 1025589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // the scanner should pass uninterpreted bodies to the RegExp 1026589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // constructor. 1027589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch LiteralScope literal(this); 1028589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (seen_equal) { 1029589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralChar('='); 1030589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1031589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1032589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (c0_ != '/' || in_character_class) { 1033589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (unicode_cache_->IsLineTerminator(c0_) || c0_ < 0) return false; 1034589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '\\') { // Escape sequence. 1035589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 1036589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (unicode_cache_->IsLineTerminator(c0_) || c0_ < 0) return false; 1037589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 1038589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // If the escape allows more characters, i.e., \x??, \u????, or \c?, 1039589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // only "safe" characters are allowed (letters, digits, underscore), 1040589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // otherwise the escape isn't valid and the invalid character has 1041589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // its normal meaning. I.e., we can just continue scanning without 1042589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // worrying whether the following characters are part of the escape 1043589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // or not, since any '/', '\\' or '[' is guaranteed to not be part 1044589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // of the escape sequence. 1045589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1046589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // TODO(896): At some point, parse RegExps more throughly to capture 1047589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // octal esacpes in strict mode. 1048589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { // Unescaped character. 1049589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '[') in_character_class = true; 1050589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == ']') in_character_class = false; 1051589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 1052589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1053589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1054589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); // consume '/' 1055589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1056589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch literal.Complete(); 1057589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1058589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return true; 105980d68eab642096c1a48b6474d6ec33064b0ad1f5Kristian Monsen} 106080d68eab642096c1a48b6474d6ec33064b0ad1f5Kristian Monsen 106180d68eab642096c1a48b6474d6ec33064b0ad1f5Kristian Monsen 106285b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdochbool JavaScriptScanner::ScanLiteralUnicodeEscape() { 1063589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ASSERT(c0_ == '\\'); 1064589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 chars_read[6] = {'\\', 'u', 0, 0, 0, 0}; 1065589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 1066589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int i = 1; 1067589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == 'u') { 1068589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch i++; 1069589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (i < 6) { 1070589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 1071589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (!IsHexDigit(c0_)) break; 1072589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch chars_read[i] = c0_; 1073589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch i++; 1074589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1075589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1076589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (i < 6) { 1077589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Incomplete escape. Undo all advances and return false. 1078589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (i > 0) { 1079589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch i--; 1080589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch PushBack(chars_read[i]); 1081589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1082589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return false; 1083589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1084589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Complete escape. Add all chars to current literal buffer. 1085589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch for (int i = 0; i < 6; i++) { 1086589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralChar(chars_read[i]); 1087589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1088589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return true; 108980d68eab642096c1a48b6474d6ec33064b0ad1f5Kristian Monsen} 109080d68eab642096c1a48b6474d6ec33064b0ad1f5Kristian Monsen 109180d68eab642096c1a48b6474d6ec33064b0ad1f5Kristian Monsen 109285b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdochbool JavaScriptScanner::ScanRegExpFlags() { 1093589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Scan regular expression flags. 1094589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch LiteralScope literal(this); 1095589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (unicode_cache_->IsIdentifierPart(c0_)) { 1096589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ != '\\') { 1097589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 1098589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 1099589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (!ScanLiteralUnicodeEscape()) { 1100589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 1101589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1102589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1103589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1104589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch literal.Complete(); 1105589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1106589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.location.end_pos = source_pos() - 1; 1107589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return true; 110880d68eab642096c1a48b6474d6ec33064b0ad1f5Kristian Monsen} 110980d68eab642096c1a48b6474d6ec33064b0ad1f5Kristian Monsen 1110a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block} } // namespace v8::internal 1111