18b112d2025046f85ef7f6be087c6129c872ebad2Ben Murdoch// Copyright 2011 the V8 project authors. All rights reserved. 2b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// Use of this source code is governed by a BSD-style license that can be 3b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// found in the LICENSE file. 4a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 5589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// Features shared by parsing and pre-parsing scanners. 6a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 7b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include <cmath> 8589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 9b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/v8.h" 10b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 11b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "include/v8stdint.h" 12b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/ast-value-factory.h" 13b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/char-predicates-inl.h" 14b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/conversions-inl.h" 15b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/list-inl.h" 16b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/parser.h" 17b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#include "src/scanner.h" 18a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 19a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Blocknamespace v8 { 20a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Blocknamespace internal { 21a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 22b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 23b8a8cc1952d61a2f3a2568848933943a543b5d3eBen MurdochHandle<String> LiteralBuffer::Internalize(Isolate* isolate) const { 24b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (is_one_byte()) { 25b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return isolate->factory()->InternalizeOneByteString(one_byte_literal()); 26b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 27b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return isolate->factory()->InternalizeTwoByteString(two_byte_literal()); 28b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 29b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 30b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 31a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block// ---------------------------------------------------------------------------- 323ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch// Scanner 33a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 343ef787dbeca8a5fb1086949cda830dccee07bfbdBen MurdochScanner::Scanner(UnicodeCache* unicode_cache) 353ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch : unicode_cache_(unicode_cache), 363ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch octal_pos_(Location::invalid()), 373ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch harmony_scoping_(false), 38b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch harmony_modules_(false), 39b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch harmony_numeric_literals_(false), 40b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch harmony_classes_(false) { } 4185b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch 4285b71799222b55eb5dd74ea26efe0c64ab655c8cBen Murdoch 433ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid Scanner::Initialize(Utf16CharacterStream* source) { 443ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch source_ = source; 453ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Need to capture identifiers in order to recognize "get" and "set" 463ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // in object literals. 473ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Init(); 483ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // Skip initial whitespace allowing HTML comment ends just like 493ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch // after a newline and scan first token. 503ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch has_line_terminator_before_next_ = true; 513ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch SkipWhiteSpace(); 523ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch Scan(); 53589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 54b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 55589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 56589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdochuc32 Scanner::ScanHexNumber(int expected_length) { 57b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(expected_length <= 4); // prevent overflow 58589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 59589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 digits[4] = { 0, 0, 0, 0 }; 60589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 x = 0; 61589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch for (int i = 0; i < expected_length; i++) { 62589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch digits[i] = c0_; 63589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int d = HexValue(c0_); 64589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (d < 0) { 65589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // According to ECMA-262, 3rd, 7.8.4, page 18, these hex escapes 66589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // should be illegal, but other JS VMs just return the 67589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // non-escaped version of the original character. 68589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 69589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Push back digits that we have advanced past. 70589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch for (int j = i-1; j >= 0; j--) { 71589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch PushBack(digits[j]); 72589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 73589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return -1; 74589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 75589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch x = x * 16 + d; 76589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 77b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch } 78589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 79589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return x; 80a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block} 81a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 82a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 83589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// Ensure that tokens can be stored in a byte. 84589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben MurdochSTATIC_ASSERT(Token::NUM_TOKENS <= 0x100); 85589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 86589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// Table of one-character tokens, by character (0x00..0x7f only). 87589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdochstatic const byte one_char_tokens[] = { 88589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 89589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 90589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 91589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 92589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 93589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 94589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 95589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 96589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 97589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 98589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 99589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 100589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 101589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 102589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 103589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 104589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 105589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 106589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 107589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 108589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 109589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 110589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 111589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 112589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 113589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 114589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 115589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 116589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 117589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 118589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 119589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 120589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 121589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 122589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 123589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 124589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 125589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 126589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 127589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 128589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::LPAREN, // 0x28 129589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::RPAREN, // 0x29 130589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 131589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 132589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::COMMA, // 0x2c 133589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 134589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 135589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 136589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 137589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 138589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 139589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 140589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 141589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 142589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 143589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 144589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 145589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 146589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::COLON, // 0x3a 147589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::SEMICOLON, // 0x3b 148589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 149589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 150589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 151589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::CONDITIONAL, // 0x3f 152589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 153589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 154589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 155589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 156589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 157589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 158589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 159589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 160589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 161589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 162589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 163589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 164589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 165589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 166589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 167589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 168589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 169589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 170589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 171589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 172589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 173589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 174589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 175589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 176589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 177589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 178589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 179589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::LBRACK, // 0x5b 180589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 181589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::RBRACK, // 0x5d 182589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 183589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 184589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 185589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 186589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 187589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 188589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 189589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 190589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 191589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 192589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 193589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 194589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 195589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 196589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 197589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 198589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 199589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 200589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 201589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 202589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 203589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 204589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 205589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 206589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 207589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 208589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 209589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 210589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 211589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::LBRACE, // 0x7b 212589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL, 213589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::RBRACE, // 0x7d 214589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::BIT_NOT, // 0x7e 215589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::ILLEGAL 216589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch}; 217589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 218589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 2193ef787dbeca8a5fb1086949cda830dccee07bfbdBen MurdochToken::Value Scanner::Next() { 220589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch current_ = next_; 221589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch has_line_terminator_before_next_ = false; 222589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch has_multiline_comment_before_next_ = false; 223589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (static_cast<unsigned>(c0_) <= 0x7f) { 224589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::Value token = static_cast<Token::Value>(one_char_tokens[c0_]); 225589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (token != Token::ILLEGAL) { 226589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int pos = source_pos(); 227589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.token = token; 228589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.location.beg_pos = pos; 229589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.location.end_pos = pos + 1; 230589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 231589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return current_.token; 232589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 233a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block } 234589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Scan(); 235589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return current_.token; 236b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 237b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 238b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 239b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch// TODO(yangguo): check whether this is actually necessary. 240b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochstatic inline bool IsLittleEndianByteOrderMark(uc32 c) { 241589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // The Unicode value U+FFFE is guaranteed never to be assigned as a 242589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Unicode character; this implies that in a Unicode context the 243589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // 0xFF, 0xFE byte pattern can only be interpreted as the U+FEFF 244589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // character expressed in little-endian byte order (since it could 245589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // not be a U+FFFE character expressed in big-endian byte 246589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // order). Nevertheless, we check for it to be compatible with 247589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Spidermonkey. 248b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return c == 0xFFFE; 249b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 250b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 251589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 2523ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochbool Scanner::SkipWhiteSpace() { 253589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int start_position = source_pos(); 254589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 255589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (true) { 256b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch while (true) { 257b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Advance as long as character is a WhiteSpace or LineTerminator. 258b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Remember if the latter is the case. 259589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (unicode_cache_->IsLineTerminator(c0_)) { 260589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch has_line_terminator_before_next_ = true; 261b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } else if (!unicode_cache_->IsWhiteSpace(c0_) && 262b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch !IsLittleEndianByteOrderMark(c0_)) { 263b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch break; 264589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 265589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 266589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 267589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 268589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // If there is an HTML comment end '-->' at the beginning of a 269589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // line (with only whitespace in front of it), we treat the rest 270589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // of the line as a comment. This is in line with the way 271589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // SpiderMonkey handles it. 272589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '-' && has_line_terminator_before_next_) { 273589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 274589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '-') { 275589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 276589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '>') { 277589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Treat the rest of the line as a comment. 278589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch SkipSingleLineComment(); 279589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Continue skipping white space after the comment. 280589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch continue; 281589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 282589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch PushBack('-'); // undo Advance() 283589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 284589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch PushBack('-'); // undo Advance() 285589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 286589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Return whether or not we skipped any characters. 287589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return source_pos() != start_position; 288589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 289b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 290b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 291b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 2923ef787dbeca8a5fb1086949cda830dccee07bfbdBen MurdochToken::Value Scanner::SkipSingleLineComment() { 293589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 294b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 295589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // The line terminator at the end of the line is not considered 296589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // to be part of the single-line comment; it is recognized 297589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // separately by the lexical grammar and becomes part of the 298589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // stream of input elements for the syntactic grammar (see 299589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // ECMA-262, section 7.4). 300589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (c0_ >= 0 && !unicode_cache_->IsLineTerminator(c0_)) { 301589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 302589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 303b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 304589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::WHITESPACE; 305a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block} 306a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 307a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 308b8a8cc1952d61a2f3a2568848933943a543b5d3eBen MurdochToken::Value Scanner::SkipSourceURLComment() { 309b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch TryToParseSourceURLComment(); 310b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch while (c0_ >= 0 && !unicode_cache_->IsLineTerminator(c0_)) { 311b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Advance(); 312b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 313b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 314b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return Token::WHITESPACE; 315b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 316b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 317b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 318b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochvoid Scanner::TryToParseSourceURLComment() { 319b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Magic comments are of the form: //[#@]\s<name>=\s*<value>\s*.* and this 320b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // function will just return if it cannot parse a magic comment. 321b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (!unicode_cache_->IsWhiteSpace(c0_)) 322b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return; 323b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Advance(); 324b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch LiteralBuffer name; 325b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch while (c0_ >= 0 && !unicode_cache_->IsWhiteSpaceOrLineTerminator(c0_) && 326b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch c0_ != '=') { 327b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch name.AddChar(c0_); 328b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Advance(); 329b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 330b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (!name.is_one_byte()) return; 331b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Vector<const uint8_t> name_literal = name.one_byte_literal(); 332b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch LiteralBuffer* value; 333b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (name_literal == STATIC_CHAR_VECTOR("sourceURL")) { 334b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch value = &source_url_; 335b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } else if (name_literal == STATIC_CHAR_VECTOR("sourceMappingURL")) { 336b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch value = &source_mapping_url_; 337b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } else { 338b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return; 339b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 340b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (c0_ != '=') 341b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return; 342b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Advance(); 343b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch value->Reset(); 344b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch while (c0_ >= 0 && unicode_cache_->IsWhiteSpace(c0_)) { 345b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Advance(); 346b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 347b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch while (c0_ >= 0 && !unicode_cache_->IsLineTerminator(c0_)) { 348b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Disallowed characters. 349b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (c0_ == '"' || c0_ == '\'') { 350b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch value->Reset(); 351b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return; 352b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 353b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (unicode_cache_->IsWhiteSpace(c0_)) { 354b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch break; 355b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 356b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch value->AddChar(c0_); 357b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Advance(); 358b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 359b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Allow whitespace at the end. 360b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch while (c0_ >= 0 && !unicode_cache_->IsLineTerminator(c0_)) { 361b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (!unicode_cache_->IsWhiteSpace(c0_)) { 362b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch value->Reset(); 363b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch break; 364b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 365b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Advance(); 366b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 367b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 368b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 369b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 3703ef787dbeca8a5fb1086949cda830dccee07bfbdBen MurdochToken::Value Scanner::SkipMultiLineComment() { 371b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(c0_ == '*'); 372589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 373589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 374589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (c0_ >= 0) { 375589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 ch = c0_; 376589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 377589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (unicode_cache_->IsLineTerminator(ch)) { 378589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Following ECMA-262, section 7.4, a comment containing 379589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // a newline will make the comment count as a line-terminator. 380589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch has_multiline_comment_before_next_ = true; 381589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 382589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // If we have reached the end of the multi-line comment, we 383589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // consume the '/' and insert a whitespace. This way all 384589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // multi-line comments are treated as whitespace. 385589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (ch == '*' && c0_ == '/') { 386589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch c0_ = ' '; 387589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::WHITESPACE; 388589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 389b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch } 390589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 391589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Unterminated multi-line comment. 392589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::ILLEGAL; 393b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 394b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 395b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 3963ef787dbeca8a5fb1086949cda830dccee07bfbdBen MurdochToken::Value Scanner::ScanHtmlComment() { 397589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Check for <!-- comments. 398b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(c0_ == '!'); 399589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 400589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '-') { 401589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 402589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '-') return SkipSingleLineComment(); 403589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch PushBack('-'); // undo Advance() 404589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 405589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch PushBack('!'); // undo Advance() 406b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(c0_ == '!'); 407589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::LT; 408b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 409b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 410b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 4113ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid Scanner::Scan() { 412589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.literal_chars = NULL; 413589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Token::Value token; 414589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch do { 415589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Remember the position of the next token 416589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.location.beg_pos = source_pos(); 417589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 418589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch switch (c0_) { 419589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case ' ': 420589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '\t': 421589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 422589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::WHITESPACE; 423589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 424589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 425589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '\n': 426589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 427589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch has_line_terminator_before_next_ = true; 428589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::WHITESPACE; 429589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 430589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 431589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '"': case '\'': 432589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = ScanString(); 433589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 434589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 435589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '<': 436589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // < <= << <<= <!-- 437589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 438589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '=') { 439589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::LTE); 440589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ == '<') { 441589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select('=', Token::ASSIGN_SHL, Token::SHL); 442589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ == '!') { 443589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = ScanHtmlComment(); 444589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 445589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::LT; 446589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 447589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 448589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 449589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '>': 450589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // > >= >> >>= >>> >>>= 451589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 452589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '=') { 453589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::GTE); 454589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ == '>') { 455589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // >> >>= >>> >>>= 456589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 457589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '=') { 458589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::ASSIGN_SAR); 459589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ == '>') { 460589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select('=', Token::ASSIGN_SHR, Token::SHR); 461589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 462589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::SAR; 463589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 464589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 465589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::GT; 466589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 467589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 468589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 469589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '=': 470b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // = == === => 471589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 472589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '=') { 473589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select('=', Token::EQ_STRICT, Token::EQ); 474b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } else if (c0_ == '>') { 475b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch token = Select(Token::ARROW); 476589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 477589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::ASSIGN; 478589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 479589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 480589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 481589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '!': 482589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // ! != !== 483589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 484589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '=') { 485589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select('=', Token::NE_STRICT, Token::NE); 486589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 487589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::NOT; 488589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 489589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 490589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 491589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '+': 492589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // + ++ += 493589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 494589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '+') { 495589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::INC); 496589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ == '=') { 497589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::ASSIGN_ADD); 498589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 499589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::ADD; 500589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 501589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 502589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 503589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '-': 504589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // - -- --> -= 505589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 506589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '-') { 507589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 508589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '>' && has_line_terminator_before_next_) { 509589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // For compatibility with SpiderMonkey, we skip lines that 510589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // start with an HTML comment end '-->'. 511589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = SkipSingleLineComment(); 512589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 513589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::DEC; 514589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 515589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ == '=') { 516589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::ASSIGN_SUB); 517589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 518589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::SUB; 519589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 520589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 521589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 522589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '*': 523589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // * *= 524589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select('=', Token::ASSIGN_MUL, Token::MUL); 525589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 526b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 527589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '%': 528589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // % %= 529589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select('=', Token::ASSIGN_MOD, Token::MOD); 530589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 531b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 532589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '/': 533589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // / // /* /= 534589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 535589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '/') { 536b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Advance(); 537b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (c0_ == '@' || c0_ == '#') { 538b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Advance(); 539b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch token = SkipSourceURLComment(); 540b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } else { 541b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch PushBack(c0_); 542b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch token = SkipSingleLineComment(); 543b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 544589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ == '*') { 545589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = SkipMultiLineComment(); 546589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ == '=') { 547589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::ASSIGN_DIV); 548589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 549589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::DIV; 550589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 551589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 552589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 553589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '&': 554589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // & && &= 555589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 556589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '&') { 557589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::AND); 558589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ == '=') { 559589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::ASSIGN_BIT_AND); 560589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 561589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::BIT_AND; 562589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 563589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 564589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 565589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '|': 566589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // | || |= 567589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 568589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '|') { 569589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::OR); 570589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ == '=') { 571589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::ASSIGN_BIT_OR); 572589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 573589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::BIT_OR; 574589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 575589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 576589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 577589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '^': 578589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // ^ ^= 579589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select('=', Token::ASSIGN_BIT_XOR, Token::BIT_XOR); 580589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 581589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 582589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '.': 583589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // . Number 584589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 585589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (IsDecimalDigit(c0_)) { 586589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = ScanNumber(true); 587589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 588589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::PERIOD; 589589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 590589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 591589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 592589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case ':': 593589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::COLON); 594589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 595589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 596589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case ';': 597589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::SEMICOLON); 598589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 599589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 600589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case ',': 601589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::COMMA); 602589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 603589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 604589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '(': 605589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::LPAREN); 606589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 607589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 608589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case ')': 609589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::RPAREN); 610589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 611589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 612589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '[': 613589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::LBRACK); 614589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 615589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 616589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case ']': 617589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::RBRACK); 618589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 619589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 620589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '{': 621589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::LBRACE); 622589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 623589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 624589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '}': 625589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::RBRACE); 626589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 627589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 628589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '?': 629589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::CONDITIONAL); 630589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 631589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 632589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '~': 633589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::BIT_NOT); 634589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 635589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 636589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch default: 637589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (unicode_cache_->IsIdentifierStart(c0_)) { 638589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = ScanIdentifierOrKeyword(); 639589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (IsDecimalDigit(c0_)) { 640589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = ScanNumber(false); 641589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (SkipWhiteSpace()) { 642589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::WHITESPACE; 643589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if (c0_ < 0) { 644589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Token::EOS; 645589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 646589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch token = Select(Token::ILLEGAL); 647589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 648589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 649589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 650589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 651589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Continue scanning for tokens as long as we're just skipping 652589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // whitespace. 653589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } while (token == Token::WHITESPACE); 654589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 655589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.location.end_pos = source_pos(); 656589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.token = token; 657b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 658b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 659b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 6603ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid Scanner::SeekForward(int pos) { 661589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // After this call, we will have the token at the given position as 662589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // the "next" token. The "current" token will be invalid. 663589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (pos == next_.location.beg_pos) return; 664589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int current_pos = source_pos(); 665b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK_EQ(next_.location.end_pos, current_pos); 666589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Positions inside the lookahead token aren't supported. 667b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(pos >= current_pos); 668589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (pos != current_pos) { 669589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch source_->SeekForward(pos - source_->pos()); 670589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 671589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // This function is only called to seek to the location 672589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // of the end of a function (at the "}" token). It doesn't matter 673589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // whether there was a line terminator in the part we skip. 674589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch has_line_terminator_before_next_ = false; 675589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch has_multiline_comment_before_next_ = false; 676b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch } 677589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Scan(); 678589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 679589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 680589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 681b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochbool Scanner::ScanEscape() { 682589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 c = c0_; 683589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 684589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 685589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Skip escaped newlines. 686589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (unicode_cache_->IsLineTerminator(c)) { 687589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Allow CR+LF newlines in multiline string literals. 688589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (IsCarriageReturn(c) && IsLineFeed(c0_)) Advance(); 689589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Allow LF+CR newlines in multiline string literals. 690589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (IsLineFeed(c) && IsCarriageReturn(c0_)) Advance(); 691b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return true; 692589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 693589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 694589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch switch (c) { 695589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '\'': // fall through 696589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '"' : // fall through 697589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '\\': break; 698589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case 'b' : c = '\b'; break; 699589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case 'f' : c = '\f'; break; 700589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case 'n' : c = '\n'; break; 701589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case 'r' : c = '\r'; break; 702589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case 't' : c = '\t'; break; 703589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case 'u' : { 704589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch c = ScanHexNumber(4); 705b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (c < 0) return false; 706589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 707589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 708589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case 'v' : c = '\v'; break; 709589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case 'x' : { 710589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch c = ScanHexNumber(2); 711b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (c < 0) return false; 712589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 713b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch } 714589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '0' : // fall through 715589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '1' : // fall through 716589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '2' : // fall through 717589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '3' : // fall through 718589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '4' : // fall through 719589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '5' : // fall through 720589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '6' : // fall through 721589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case '7' : c = ScanOctalEscape(c, 2); break; 722b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch } 723589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 724b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // According to ECMA-262, section 7.8.4, characters not covered by the 725b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // above cases should be illegal, but they are commonly handled as 726b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // non-escaped characters by JS VMs. 727589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralChar(c); 728b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return true; 729b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 730b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 731b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 732589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// Octal escapes of the forms '\0xx' and '\xxx' are not a part of 733589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// ECMA-262. Other JS VMs support them. 7343ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochuc32 Scanner::ScanOctalEscape(uc32 c, int length) { 735589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 x = c - '0'; 736589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int i = 0; 737589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch for (; i < length; i++) { 738589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int d = c0_ - '0'; 739589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (d < 0 || d > 7) break; 740589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int nx = x * 8 + d; 741589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (nx >= 256) break; 742589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch x = nx; 743589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 744589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 745589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Anything except '\0' is an octal escape sequence, illegal in strict mode. 746589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Remember the position of octal escape sequences so that an error 747589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // can be reported later (in strict mode). 748589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // We don't report the error immediately, because the octal escape can 749589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // occur before the "use strict" directive. 750589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c != '0' || i > 0) { 751589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch octal_pos_ = Location(source_pos() - i - 1, source_pos() - 1); 752589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 753589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return x; 754589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 755589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 756589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 7573ef787dbeca8a5fb1086949cda830dccee07bfbdBen MurdochToken::Value Scanner::ScanString() { 758589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 quote = c0_; 759589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); // consume quote 760b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 761589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch LiteralScope literal(this); 762589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (c0_ != quote && c0_ >= 0 763589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch && !unicode_cache_->IsLineTerminator(c0_)) { 764589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 c = c0_; 765589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 766589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c == '\\') { 767b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (c0_ < 0 || !ScanEscape()) return Token::ILLEGAL; 768589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 769589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralChar(c); 770589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 771589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 772589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ != quote) return Token::ILLEGAL; 773589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch literal.Complete(); 774b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 775589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); // consume quote 776589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::STRING; 777b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 778b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 779b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 7803ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochvoid Scanner::ScanDecimalDigits() { 781589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (IsDecimalDigit(c0_)) 782589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 783b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 784b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 785b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 7863ef787dbeca8a5fb1086949cda830dccee07bfbdBen MurdochToken::Value Scanner::ScanNumber(bool seen_period) { 787b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(IsDecimalDigit(c0_)); // the first digit of the number or the fraction 788589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 789b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch enum { DECIMAL, HEX, OCTAL, IMPLICIT_OCTAL, BINARY } kind = DECIMAL; 790589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 791589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch LiteralScope literal(this); 792589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (seen_period) { 793589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // we have already seen a decimal point of the float 794589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralChar('.'); 795589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ScanDecimalDigits(); // we know we have at least one digit 796589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 797589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 798589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // if the first character is '0' we must check for octals and hex 799589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '0') { 800589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int start_pos = source_pos(); // For reporting octal positions. 801589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 802589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 803b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // either 0, 0exxx, 0Exxx, 0.xxx, a hex number, a binary number or 804b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // an octal number. 805589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == 'x' || c0_ == 'X') { 806589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // hex number 807589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch kind = HEX; 808589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 809589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (!IsHexDigit(c0_)) { 810589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // we must have at least one hex digit after 'x'/'X' 811589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::ILLEGAL; 812589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 813589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (IsHexDigit(c0_)) { 814589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 815589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 816b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } else if (harmony_numeric_literals_ && (c0_ == 'o' || c0_ == 'O')) { 817b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch kind = OCTAL; 818b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch AddLiteralCharAdvance(); 819b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (!IsOctalDigit(c0_)) { 820b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // we must have at least one octal digit after 'o'/'O' 821b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return Token::ILLEGAL; 822b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 823b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch while (IsOctalDigit(c0_)) { 824b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch AddLiteralCharAdvance(); 825b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 826b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } else if (harmony_numeric_literals_ && (c0_ == 'b' || c0_ == 'B')) { 827b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch kind = BINARY; 828b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch AddLiteralCharAdvance(); 829b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (!IsBinaryDigit(c0_)) { 830b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // we must have at least one binary digit after 'b'/'B' 831b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return Token::ILLEGAL; 832b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 833b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch while (IsBinaryDigit(c0_)) { 834b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch AddLiteralCharAdvance(); 835b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 836589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else if ('0' <= c0_ && c0_ <= '7') { 837589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // (possible) octal number 838b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch kind = IMPLICIT_OCTAL; 839589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (true) { 840589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '8' || c0_ == '9') { 841589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch kind = DECIMAL; 842589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 843589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 844589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ < '0' || '7' < c0_) { 845589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Octal literal finished. 846589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch octal_pos_ = Location(start_pos, source_pos()); 847589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 848589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 849589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 850589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 851589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 852589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 853589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 854589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Parse decimal digits and allow trailing fractional part. 855589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (kind == DECIMAL) { 856589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ScanDecimalDigits(); // optional 857589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '.') { 858589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 859589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ScanDecimalDigits(); // optional 860589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 861589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 862b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch } 863589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 864589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // scan exponent, if any 865589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == 'e' || c0_ == 'E') { 866b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(kind != HEX); // 'e'/'E' must be scanned as part of the hex number 867b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (kind != DECIMAL) return Token::ILLEGAL; 868589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // scan exponent 869589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 870589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '+' || c0_ == '-') 871589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 872589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (!IsDecimalDigit(c0_)) { 873589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // we must have at least one decimal digit after 'e'/'E' 874589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::ILLEGAL; 875589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 876589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch ScanDecimalDigits(); 877589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 878589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 879589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // The source character immediately following a numeric literal must 880589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // not be an identifier start or a decimal digit; see ECMA-262 881589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // section 7.8.3, page 17 (note that we read only one decimal digit 882589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // if the value is 0). 883589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (IsDecimalDigit(c0_) || unicode_cache_->IsIdentifierStart(c0_)) 884589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::ILLEGAL; 885589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 886589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch literal.Complete(); 887589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 888589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::NUMBER; 889b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 890b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 891b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 8923ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochuc32 Scanner::ScanIdentifierUnicodeEscape() { 893589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 894589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ != 'u') return -1; 895589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 896589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 result = ScanHexNumber(4); 897589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (result < 0) PushBack('u'); 898589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return result; 899589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch} 900589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 901589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 902589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// ---------------------------------------------------------------------------- 903589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch// Keyword Matcher 904589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 905b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch#define KEYWORDS(KEYWORD_GROUP, KEYWORD) \ 906b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD_GROUP('b') \ 907b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("break", Token::BREAK) \ 908b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD_GROUP('c') \ 909b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("case", Token::CASE) \ 910b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("catch", Token::CATCH) \ 911b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("class", \ 912b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch harmony_classes ? Token::CLASS : Token::FUTURE_RESERVED_WORD) \ 913b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("const", Token::CONST) \ 914b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("continue", Token::CONTINUE) \ 915b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD_GROUP('d') \ 916b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("debugger", Token::DEBUGGER) \ 917b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("default", Token::DEFAULT) \ 918b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("delete", Token::DELETE) \ 919b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("do", Token::DO) \ 920b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD_GROUP('e') \ 921b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("else", Token::ELSE) \ 922b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("enum", Token::FUTURE_RESERVED_WORD) \ 923b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("export", \ 924b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch harmony_modules ? Token::EXPORT : Token::FUTURE_RESERVED_WORD) \ 925b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("extends", \ 926b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch harmony_classes ? Token::EXTENDS : Token::FUTURE_RESERVED_WORD) \ 927b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD_GROUP('f') \ 928b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("false", Token::FALSE_LITERAL) \ 929b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("finally", Token::FINALLY) \ 930b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("for", Token::FOR) \ 931b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("function", Token::FUNCTION) \ 932b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD_GROUP('i') \ 933b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("if", Token::IF) \ 934b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("implements", Token::FUTURE_STRICT_RESERVED_WORD) \ 935b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("import", \ 936b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch harmony_modules ? Token::IMPORT : Token::FUTURE_RESERVED_WORD) \ 937b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("in", Token::IN) \ 938b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("instanceof", Token::INSTANCEOF) \ 939b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("interface", Token::FUTURE_STRICT_RESERVED_WORD) \ 940b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD_GROUP('l') \ 941b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("let", \ 942b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch harmony_scoping ? Token::LET : Token::FUTURE_STRICT_RESERVED_WORD) \ 943b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD_GROUP('n') \ 944b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("new", Token::NEW) \ 945b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("null", Token::NULL_LITERAL) \ 946b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD_GROUP('p') \ 947b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("package", Token::FUTURE_STRICT_RESERVED_WORD) \ 948b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("private", Token::FUTURE_STRICT_RESERVED_WORD) \ 949b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("protected", Token::FUTURE_STRICT_RESERVED_WORD) \ 950b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("public", Token::FUTURE_STRICT_RESERVED_WORD) \ 951b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD_GROUP('r') \ 952b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("return", Token::RETURN) \ 953b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD_GROUP('s') \ 954b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("static", harmony_classes ? Token::STATIC \ 955b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch : Token::FUTURE_STRICT_RESERVED_WORD) \ 956b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("super", \ 957b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch harmony_classes ? Token::SUPER : Token::FUTURE_RESERVED_WORD) \ 958b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("switch", Token::SWITCH) \ 959b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD_GROUP('t') \ 960b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("this", Token::THIS) \ 961b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("throw", Token::THROW) \ 962b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("true", Token::TRUE_LITERAL) \ 963b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("try", Token::TRY) \ 964b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("typeof", Token::TYPEOF) \ 965b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD_GROUP('v') \ 966b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("var", Token::VAR) \ 967b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("void", Token::VOID) \ 968b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD_GROUP('w') \ 969b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("while", Token::WHILE) \ 970b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("with", Token::WITH) \ 971b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD_GROUP('y') \ 972b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KEYWORD("yield", Token::YIELD) 973b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 974b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 975b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochstatic Token::Value KeywordOrIdentifierToken(const uint8_t* input, 976589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int input_length, 9773ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch bool harmony_scoping, 978b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool harmony_modules, 979b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool harmony_classes) { 980b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(input_length >= 1); 981589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch const int kMinLength = 2; 982589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch const int kMaxLength = 10; 983589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (input_length < kMinLength || input_length > kMaxLength) { 984589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::IDENTIFIER; 985b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch } 986589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch switch (input[0]) { 987589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch default: 988589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch#define KEYWORD_GROUP_CASE(ch) \ 989589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; \ 990589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch case ch: 991589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch#define KEYWORD(keyword, token) \ 992589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch { \ 993589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch /* 'keyword' is a char array, so sizeof(keyword) is */ \ 994589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch /* strlen(keyword) plus 1 for the NUL char. */ \ 995589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch const int keyword_length = sizeof(keyword) - 1; \ 996589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch STATIC_ASSERT(keyword_length >= kMinLength); \ 997589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch STATIC_ASSERT(keyword_length <= kMaxLength); \ 998589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (input_length == keyword_length && \ 999589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch input[1] == keyword[1] && \ 1000589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch (keyword_length <= 2 || input[2] == keyword[2]) && \ 1001589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch (keyword_length <= 3 || input[3] == keyword[3]) && \ 1002589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch (keyword_length <= 4 || input[4] == keyword[4]) && \ 1003589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch (keyword_length <= 5 || input[5] == keyword[5]) && \ 1004589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch (keyword_length <= 6 || input[6] == keyword[6]) && \ 1005589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch (keyword_length <= 7 || input[7] == keyword[7]) && \ 1006589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch (keyword_length <= 8 || input[8] == keyword[8]) && \ 1007589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch (keyword_length <= 9 || input[9] == keyword[9])) { \ 1008589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return token; \ 1009589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } \ 1010589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1011589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch KEYWORDS(KEYWORD_GROUP_CASE, KEYWORD) 1012589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1013589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::IDENTIFIER; 1014b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 1015b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 1016b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 1017b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochbool Scanner::IdentifierIsFutureStrictReserved( 1018b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch const AstRawString* string) const { 1019b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Keywords are always 1-byte strings. 1020b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return string->is_one_byte() && 1021b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Token::FUTURE_STRICT_RESERVED_WORD == 1022b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch KeywordOrIdentifierToken(string->raw_data(), string->length(), 1023b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch harmony_scoping_, harmony_modules_, 1024b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch harmony_classes_); 1025b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 1026b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1027b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 10283ef787dbeca8a5fb1086949cda830dccee07bfbdBen MurdochToken::Value Scanner::ScanIdentifierOrKeyword() { 1029b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(unicode_cache_->IsIdentifierStart(c0_)); 1030589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch LiteralScope literal(this); 1031589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Scan identifier start character. 1032589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '\\') { 1033589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 c = ScanIdentifierUnicodeEscape(); 1034589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Only allow legal identifier start characters. 1035589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c < 0 || 1036589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch c == '\\' || // No recursive escapes. 1037589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch !unicode_cache_->IsIdentifierStart(c)) { 1038589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::ILLEGAL; 1039589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1040589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralChar(c); 1041589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return ScanIdentifierSuffix(&literal); 1042b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch } 1043589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1044589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 first_char = c0_; 1045589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 1046589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralChar(first_char); 1047589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1048589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Scan the rest of the identifier characters. 1049589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (unicode_cache_->IsIdentifierPart(c0_)) { 1050589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ != '\\') { 1051589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 next_char = c0_; 1052589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 1053589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralChar(next_char); 1054589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch continue; 1055589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1056589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Fallthrough if no longer able to complete keyword. 1057589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return ScanIdentifierSuffix(&literal); 1058b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch } 1059589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1060589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch literal.Complete(); 1061589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1062b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (next_.literal_chars->is_one_byte()) { 1063b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Vector<const uint8_t> chars = next_.literal_chars->one_byte_literal(); 1064589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return KeywordOrIdentifierToken(chars.start(), 1065589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch chars.length(), 10663ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdoch harmony_scoping_, 1067b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch harmony_modules_, 1068b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch harmony_classes_); 1069589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1070589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1071589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::IDENTIFIER; 1072b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch} 1073b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 1074b0fe1620dcb4135ac3ab2d66ff93072373911299Ben Murdoch 10753ef787dbeca8a5fb1086949cda830dccee07bfbdBen MurdochToken::Value Scanner::ScanIdentifierSuffix(LiteralScope* literal) { 1076589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Scan the rest of the identifier characters. 1077589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (unicode_cache_->IsIdentifierPart(c0_)) { 1078589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '\\') { 1079589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 c = ScanIdentifierUnicodeEscape(); 1080589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Only allow legal identifier part characters. 1081589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c < 0 || 1082589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch c == '\\' || 1083589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch !unicode_cache_->IsIdentifierPart(c)) { 1084589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::ILLEGAL; 1085589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1086589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralChar(c); 1087589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 1088589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralChar(c0_); 1089589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 1090589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1091589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1092589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch literal->Complete(); 1093589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1094589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return Token::IDENTIFIER; 1095a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block} 1096a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 1097a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block 10983ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochbool Scanner::ScanRegExpPattern(bool seen_equal) { 1099589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Scan: ('/' | '/=') RegularExpressionBody '/' RegularExpressionFlags 1100589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch bool in_character_class = false; 110180d68eab642096c1a48b6474d6ec33064b0ad1f5Kristian Monsen 1102589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Previous token is either '/' or '/=', in the second case, the 1103589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // pattern starts at =. 1104589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.location.beg_pos = source_pos() - (seen_equal ? 2 : 1); 1105589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.location.end_pos = source_pos() - (seen_equal ? 1 : 0); 1106589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1107589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Scan regular expression body: According to ECMA-262, 3rd, 7.8.5, 1108589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // the scanner should pass uninterpreted bodies to the RegExp 1109589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // constructor. 1110589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch LiteralScope literal(this); 1111589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (seen_equal) { 1112589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralChar('='); 1113589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1114589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1115589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (c0_ != '/' || in_character_class) { 1116589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (unicode_cache_->IsLineTerminator(c0_) || c0_ < 0) return false; 1117589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '\\') { // Escape sequence. 1118589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 1119589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (unicode_cache_->IsLineTerminator(c0_) || c0_ < 0) return false; 1120589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 1121589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // If the escape allows more characters, i.e., \x??, \u????, or \c?, 1122589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // only "safe" characters are allowed (letters, digits, underscore), 1123589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // otherwise the escape isn't valid and the invalid character has 1124589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // its normal meaning. I.e., we can just continue scanning without 1125589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // worrying whether the following characters are part of the escape 1126589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // or not, since any '/', '\\' or '[' is guaranteed to not be part 1127589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // of the escape sequence. 1128589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1129589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // TODO(896): At some point, parse RegExps more throughly to capture 1130589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // octal esacpes in strict mode. 1131589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { // Unescaped character. 1132589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == '[') in_character_class = true; 1133589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == ']') in_character_class = false; 1134589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 1135589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1136589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1137589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); // consume '/' 1138589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1139589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch literal.Complete(); 1140589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1141589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return true; 114280d68eab642096c1a48b6474d6ec33064b0ad1f5Kristian Monsen} 114380d68eab642096c1a48b6474d6ec33064b0ad1f5Kristian Monsen 114480d68eab642096c1a48b6474d6ec33064b0ad1f5Kristian Monsen 11453ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochbool Scanner::ScanLiteralUnicodeEscape() { 1146b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(c0_ == '\\'); 1147589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch uc32 chars_read[6] = {'\\', 'u', 0, 0, 0, 0}; 1148589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 1149589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch int i = 1; 1150589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ == 'u') { 1151589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch i++; 1152589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (i < 6) { 1153589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch Advance(); 1154589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (!IsHexDigit(c0_)) break; 1155589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch chars_read[i] = c0_; 1156589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch i++; 1157589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1158589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1159589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (i < 6) { 1160589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Incomplete escape. Undo all advances and return false. 1161589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (i > 0) { 1162589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch i--; 1163589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch PushBack(chars_read[i]); 1164589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1165589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return false; 1166589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1167589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Complete escape. Add all chars to current literal buffer. 1168589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch for (int i = 0; i < 6; i++) { 1169589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralChar(chars_read[i]); 1170589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1171589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return true; 117280d68eab642096c1a48b6474d6ec33064b0ad1f5Kristian Monsen} 117380d68eab642096c1a48b6474d6ec33064b0ad1f5Kristian Monsen 117480d68eab642096c1a48b6474d6ec33064b0ad1f5Kristian Monsen 11753ef787dbeca8a5fb1086949cda830dccee07bfbdBen Murdochbool Scanner::ScanRegExpFlags() { 1176589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch // Scan regular expression flags. 1177589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch LiteralScope literal(this); 1178589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch while (unicode_cache_->IsIdentifierPart(c0_)) { 1179589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (c0_ != '\\') { 1180589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch AddLiteralCharAdvance(); 1181589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } else { 1182589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch if (!ScanLiteralUnicodeEscape()) { 1183589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch break; 1184589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1185b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Advance(); 1186589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1187589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch } 1188589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch literal.Complete(); 1189589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch 1190589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch next_.location.end_pos = source_pos() - 1; 1191589d6979ff2ef66fca2d8fa51404c369ca5e9250Ben Murdoch return true; 119280d68eab642096c1a48b6474d6ec33064b0ad1f5Kristian Monsen} 119380d68eab642096c1a48b6474d6ec33064b0ad1f5Kristian Monsen 1194b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1195b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochconst AstRawString* Scanner::CurrentSymbol(AstValueFactory* ast_value_factory) { 1196b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (is_literal_one_byte()) { 1197b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return ast_value_factory->GetOneByteString(literal_one_byte_string()); 1198b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 1199b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return ast_value_factory->GetTwoByteString(literal_two_byte_string()); 1200b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 1201b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1202b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1203b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochconst AstRawString* Scanner::NextSymbol(AstValueFactory* ast_value_factory) { 1204b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (is_next_literal_one_byte()) { 1205b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return ast_value_factory->GetOneByteString(next_literal_one_byte_string()); 1206b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 1207b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return ast_value_factory->GetTwoByteString(next_literal_two_byte_string()); 1208b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 1209b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1210b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1211b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochdouble Scanner::DoubleValue() { 1212b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(is_literal_one_byte()); 1213b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return StringToDouble( 1214b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch unicode_cache_, 1215b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch literal_one_byte_string(), 1216b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch ALLOW_HEX | ALLOW_OCTAL | ALLOW_IMPLICIT_OCTAL | ALLOW_BINARY); 1217b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 1218b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1219b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1220b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochint Scanner::FindNumber(DuplicateFinder* finder, int value) { 1221b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return finder->AddNumber(literal_one_byte_string(), value); 1222b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 1223b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1224b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1225b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochint Scanner::FindSymbol(DuplicateFinder* finder, int value) { 1226b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (is_literal_one_byte()) { 1227b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return finder->AddOneByteSymbol(literal_one_byte_string(), value); 1228b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 1229b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return finder->AddTwoByteSymbol(literal_two_byte_string(), value); 1230b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 1231b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1232b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1233b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochint DuplicateFinder::AddOneByteSymbol(Vector<const uint8_t> key, int value) { 1234b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return AddSymbol(key, true, value); 1235b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 1236b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1237b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1238b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochint DuplicateFinder::AddTwoByteSymbol(Vector<const uint16_t> key, int value) { 1239b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return AddSymbol(Vector<const uint8_t>::cast(key), false, value); 1240b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 1241b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1242b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1243b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochint DuplicateFinder::AddSymbol(Vector<const uint8_t> key, 1244b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool is_one_byte, 1245b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int value) { 1246b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch uint32_t hash = Hash(key, is_one_byte); 1247b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch byte* encoding = BackupKey(key, is_one_byte); 1248b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch HashMap::Entry* entry = map_.Lookup(encoding, hash, true); 1249b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int old_value = static_cast<int>(reinterpret_cast<intptr_t>(entry->value)); 1250b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch entry->value = 1251b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch reinterpret_cast<void*>(static_cast<intptr_t>(value | old_value)); 1252b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return old_value; 1253b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 1254b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1255b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1256b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochint DuplicateFinder::AddNumber(Vector<const uint8_t> key, int value) { 1257b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch DCHECK(key.length() > 0); 1258b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Quick check for already being in canonical form. 1259b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (IsNumberCanonical(key)) { 1260b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return AddOneByteSymbol(key, value); 1261b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 1262b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1263b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int flags = ALLOW_HEX | ALLOW_OCTAL | ALLOW_IMPLICIT_OCTAL | ALLOW_BINARY; 1264b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch double double_value = StringToDouble( 1265b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch unicode_constants_, key, flags, 0.0); 1266b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int length; 1267b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch const char* string; 1268b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (!std::isfinite(double_value)) { 1269b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch string = "Infinity"; 1270b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch length = 8; // strlen("Infinity"); 1271b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } else { 1272b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch string = DoubleToCString(double_value, 1273b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch Vector<char>(number_buffer_, kBufferSize)); 1274b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch length = StrLength(string); 1275b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 1276b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return AddSymbol(Vector<const byte>(reinterpret_cast<const byte*>(string), 1277b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch length), true, value); 1278b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 1279b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1280b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1281b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochbool DuplicateFinder::IsNumberCanonical(Vector<const uint8_t> number) { 1282b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Test for a safe approximation of number literals that are already 1283b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // in canonical form: max 15 digits, no leading zeroes, except an 1284b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // integer part that is a single zero, and no trailing zeros below 1285b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // the decimal point. 1286b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int pos = 0; 1287b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int length = number.length(); 1288b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (number.length() > 15) return false; 1289b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (number[pos] == '0') { 1290b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch pos++; 1291b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } else { 1292b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch while (pos < length && 1293b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch static_cast<unsigned>(number[pos] - '0') <= ('9' - '0')) pos++; 1294b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 1295b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (length == pos) return true; 1296b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (number[pos] != '.') return false; 1297b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch pos++; 1298b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool invalid_last_digit = true; 1299b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch while (pos < length) { 1300b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch uint8_t digit = number[pos] - '0'; 1301b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (digit > '9' - '0') return false; 1302b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch invalid_last_digit = (digit == 0); 1303b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch pos++; 1304b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 1305b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return !invalid_last_digit; 1306b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 1307b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1308b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1309b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochuint32_t DuplicateFinder::Hash(Vector<const uint8_t> key, bool is_one_byte) { 1310b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Primitive hash function, almost identical to the one used 1311b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // for strings (except that it's seeded by the length and representation). 1312b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int length = key.length(); 1313b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch uint32_t hash = (length << 1) | (is_one_byte ? 1 : 0) ; 1314b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for (int i = 0; i < length; i++) { 1315b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch uint32_t c = key[i]; 1316b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch hash = (hash + c) * 1025; 1317b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch hash ^= (hash >> 6); 1318b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 1319b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return hash; 1320b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 1321b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1322b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1323b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochbool DuplicateFinder::Match(void* first, void* second) { 1324b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Decode lengths. 1325b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Length + representation is encoded as base 128, most significant heptet 1326b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // first, with a 8th bit being non-zero while there are more heptets. 1327b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // The value encodes the number of bytes following, and whether the original 1328b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // was Latin1. 1329b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch byte* s1 = reinterpret_cast<byte*>(first); 1330b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch byte* s2 = reinterpret_cast<byte*>(second); 1331b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch uint32_t length_one_byte_field = 0; 1332b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch byte c1; 1333b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch do { 1334b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch c1 = *s1; 1335b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (c1 != *s2) return false; 1336b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch length_one_byte_field = (length_one_byte_field << 7) | (c1 & 0x7f); 1337b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch s1++; 1338b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch s2++; 1339b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } while ((c1 & 0x80) != 0); 1340b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch int length = static_cast<int>(length_one_byte_field >> 1); 1341b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return memcmp(s1, s2, length) == 0; 1342b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 1343b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1344b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1345b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochbyte* DuplicateFinder::BackupKey(Vector<const uint8_t> bytes, 1346b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch bool is_one_byte) { 1347b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch uint32_t one_byte_length = (bytes.length() << 1) | (is_one_byte ? 1 : 0); 1348b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch backing_store_.StartSequence(); 1349b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // Emit one_byte_length as base-128 encoded number, with the 7th bit set 1350b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch // on the byte of every heptet except the last, least significant, one. 1351b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (one_byte_length >= (1 << 7)) { 1352b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (one_byte_length >= (1 << 14)) { 1353b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (one_byte_length >= (1 << 21)) { 1354b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if (one_byte_length >= (1 << 28)) { 1355b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch backing_store_.Add( 1356b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch static_cast<uint8_t>((one_byte_length >> 28) | 0x80)); 1357b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 1358b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch backing_store_.Add( 1359b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch static_cast<uint8_t>((one_byte_length >> 21) | 0x80u)); 1360b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 1361b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch backing_store_.Add( 1362b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch static_cast<uint8_t>((one_byte_length >> 14) | 0x80u)); 1363b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 1364b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch backing_store_.Add(static_cast<uint8_t>((one_byte_length >> 7) | 0x80u)); 1365b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch } 1366b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch backing_store_.Add(static_cast<uint8_t>(one_byte_length & 0x7f)); 1367b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1368b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch backing_store_.AddBlock(bytes); 1369b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return backing_store_.EndSequence().start(); 1370b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch} 1371b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 1372a7e24c173cf37484693b9abb38e494fa7bd7baebSteve Block} } // namespace v8::internal 1373