18e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project/* 28e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project * Copyright (C) 1999-2000 Harri Porten (porten@kde.org) 35f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian * Copyright (C) 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 Apple Inc. All rights reserved. 4ca9cb53ed1119a3fd98fafa0972ffeb56dee1c24Steve Block * Copyright (C) 2010 Zoltan Herczeg (zherczeg@inf.u-szeged.hu) 58e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project * 68e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project * This library is free software; you can redistribute it and/or 78e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project * modify it under the terms of the GNU Library General Public 88e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project * License as published by the Free Software Foundation; either 98e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project * version 2 of the License, or (at your option) any later version. 108e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project * 118e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project * This library is distributed in the hope that it will be useful, 128e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project * but WITHOUT ANY WARRANTY; without even the implied warranty of 138e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 148e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project * Library General Public License for more details. 158e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project * 168e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project * You should have received a copy of the GNU Library General Public License 178e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project * along with this library; see the file COPYING.LIB. If not, write to 188e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project * the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, 198e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project * Boston, MA 02110-1301, USA. 208e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project * 218e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project */ 228e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project 238e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project#ifndef Lexer_h 248e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project#define Lexer_h 258e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project 26e458d70a0d18538346f41b503114c9ebe6b2ce12Leon Clarke#include "JSParser.h" 27635860845790a19bf50bbc51ba8fb66a96dde068The Android Open Source Project#include "Lookup.h" 28231d4e3152a9c27a73b6ac7badbe6be673aa3ddfSteve Block#include "ParserArena.h" 298e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project#include "SourceCode.h" 305f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian#include <wtf/ASCIICType.h> 31ca9cb53ed1119a3fd98fafa0972ffeb56dee1c24Steve Block#include <wtf/AlwaysInline.h> 320bf48ef3be53ddaa52bbead65dfd75bf90e7a2b5Ben Murdoch#include <wtf/SegmentedVector.h> 33635860845790a19bf50bbc51ba8fb66a96dde068The Android Open Source Project#include <wtf/Vector.h> 348f72e70a9fd78eec56623b3a62e68f16b7b27e28Feng Qian#include <wtf/unicode/Unicode.h> 358e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project 368e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Projectnamespace JSC { 378e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project 388e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project class RegExp; 398e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project 40ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch class Lexer { 41ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch WTF_MAKE_NONCOPYABLE(Lexer); WTF_MAKE_FAST_ALLOCATED; 428e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project public: 435f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian // Character manipulation functions. 445f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian static bool isWhiteSpace(int character); 455f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian static bool isLineTerminator(int character); 465f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian static unsigned char convertHex(int c1, int c2); 475f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian static UChar convertUnicode(int c1, int c2, int c3, int c4); 485f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian 495f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian // Functions to set up parsing. 50231d4e3152a9c27a73b6ac7badbe6be673aa3ddfSteve Block void setCode(const SourceCode&, ParserArena&); 51635860845790a19bf50bbc51ba8fb66a96dde068The Android Open Source Project void setIsReparsing() { m_isReparsing = true; } 52a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch bool isReparsing() const { return m_isReparsing; } 538e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project 545f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian // Functions for the parser itself. 55967717af5423377c967781471ee106e2bb4e11c8Ben Murdoch enum LexType { IdentifyReservedWords, IgnoreReservedWords }; 56a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch JSTokenType lex(JSTokenData* lvalp, JSTokenInfo* llocp, LexType, bool strictMode); 5765f03d4f644ce73618e5f4f50dd694b26f55ae12Ben Murdoch bool nextTokenIsColon(); 585f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian int lineNumber() const { return m_lineNumber; } 5906ea8e899e48f1f2f396b70e63fae369f2f23232Kristian Monsen void setLastLineNumber(int lastLineNumber) { m_lastLineNumber = lastLineNumber; } 6006ea8e899e48f1f2f396b70e63fae369f2f23232Kristian Monsen int lastLineNumber() const { return m_lastLineNumber; } 618e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project bool prevTerminator() const { return m_terminator; } 625f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian SourceCode sourceCode(int openBrace, int closeBrace, int firstLine); 63231d4e3152a9c27a73b6ac7badbe6be673aa3ddfSteve Block bool scanRegExp(const Identifier*& pattern, const Identifier*& flags, UChar patternPrefix = 0); 64231d4e3152a9c27a73b6ac7badbe6be673aa3ddfSteve Block bool skipRegExp(); 658e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project 665f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian // Functions for use after parsing. 678e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project bool sawError() const { return m_error; } 688e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project void clear(); 69e458d70a0d18538346f41b503114c9ebe6b2ce12Leon Clarke int currentOffset() { return m_code - m_codeStart; } 70e458d70a0d18538346f41b503114c9ebe6b2ce12Leon Clarke void setOffset(int offset) 71e458d70a0d18538346f41b503114c9ebe6b2ce12Leon Clarke { 7265f03d4f644ce73618e5f4f50dd694b26f55ae12Ben Murdoch m_error = 0; 73e458d70a0d18538346f41b503114c9ebe6b2ce12Leon Clarke m_code = m_codeStart + offset; 7465f03d4f644ce73618e5f4f50dd694b26f55ae12Ben Murdoch m_buffer8.resize(0); 7565f03d4f644ce73618e5f4f50dd694b26f55ae12Ben Murdoch m_buffer16.resize(0); 762daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch // Faster than an if-else sequence 772daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch m_current = -1; 782daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch if (LIKELY(m_code < m_codeEnd)) 792daae5fd11344eaa88a0d92b0f6d65f8d2255c00Ben Murdoch m_current = *m_code; 80e458d70a0d18538346f41b503114c9ebe6b2ce12Leon Clarke } 81ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch void setLineNumber(int line) 82ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch { 83ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch m_lineNumber = line; 84ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch } 85ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch 86ab9e7a118cf1ea2e3a93dce683b2ded3e7291ddbBen Murdoch SourceProvider* sourceProvider() const { return m_source->provider(); } 878f72e70a9fd78eec56623b3a62e68f16b7b27e28Feng Qian 888e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project private: 898e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project friend class JSGlobalData; 905f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian 918e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project Lexer(JSGlobalData*); 928e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project ~Lexer(); 938e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project 948e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project void record8(int); 958e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project void record16(int); 968e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project void record16(UChar); 978e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project 985f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian void copyCodeWithoutBOMs(); 995f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian 100ca9cb53ed1119a3fd98fafa0972ffeb56dee1c24Steve Block ALWAYS_INLINE void shift(); 101ca9cb53ed1119a3fd98fafa0972ffeb56dee1c24Steve Block ALWAYS_INLINE int peek(int offset); 102ca9cb53ed1119a3fd98fafa0972ffeb56dee1c24Steve Block int getUnicodeCharacter(); 103ca9cb53ed1119a3fd98fafa0972ffeb56dee1c24Steve Block void shiftLineTerminator(); 104ca9cb53ed1119a3fd98fafa0972ffeb56dee1c24Steve Block 105ca9cb53ed1119a3fd98fafa0972ffeb56dee1c24Steve Block ALWAYS_INLINE const UChar* currentCharacter() const; 106ca9cb53ed1119a3fd98fafa0972ffeb56dee1c24Steve Block ALWAYS_INLINE int currentOffset() const; 1075f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian 108ca9cb53ed1119a3fd98fafa0972ffeb56dee1c24Steve Block ALWAYS_INLINE const Identifier* makeIdentifier(const UChar* characters, size_t length); 1095f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian 110ca9cb53ed1119a3fd98fafa0972ffeb56dee1c24Steve Block ALWAYS_INLINE bool lastTokenWasRestrKeyword() const; 111635860845790a19bf50bbc51ba8fb66a96dde068The Android Open Source Project 112dd8bb3de4f353a81954234999f1fea748aee2ea9Ben Murdoch ALWAYS_INLINE JSTokenType parseIdentifier(JSTokenData*, LexType); 113a94275402997c11dd2e778633dacf4b7e630a35dBen Murdoch ALWAYS_INLINE bool parseString(JSTokenData* lvalp, bool strictMode); 1145ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen ALWAYS_INLINE void parseHex(double& returnValue); 1155ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen ALWAYS_INLINE bool parseOctal(double& returnValue); 1165ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen ALWAYS_INLINE bool parseDecimal(double& returnValue); 1175ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen ALWAYS_INLINE void parseNumberAfterDecimalPoint(); 1185ddde30071f639962dd557c453f2ad01f8f0fd00Kristian Monsen ALWAYS_INLINE bool parseNumberAfterExponentIndicator(); 1195abb8606fa57c3ebfc8b3c3dbc3fa4a25d2ae306Iain Merrick ALWAYS_INLINE bool parseMultilineComment(); 120e458d70a0d18538346f41b503114c9ebe6b2ce12Leon Clarke 121635860845790a19bf50bbc51ba8fb66a96dde068The Android Open Source Project static const size_t initialReadBufferCapacity = 32; 122e458d70a0d18538346f41b503114c9ebe6b2ce12Leon Clarke 1235f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian int m_lineNumber; 12406ea8e899e48f1f2f396b70e63fae369f2f23232Kristian Monsen int m_lastLineNumber; 1258e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project 1268e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project Vector<char> m_buffer8; 1278e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project Vector<UChar> m_buffer16; 1288e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project bool m_terminator; 1298e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project bool m_delimited; // encountered delimiter like "'" and "}" on last run 1308e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project int m_lastToken; 1318e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project 1328e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project const SourceCode* m_source; 1338e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project const UChar* m_code; 1345f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian const UChar* m_codeStart; 1355f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian const UChar* m_codeEnd; 136635860845790a19bf50bbc51ba8fb66a96dde068The Android Open Source Project bool m_isReparsing; 1375f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian bool m_atLineStart; 1388e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project bool m_error; 1398e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project 1408e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project // current and following unicode characters (int to allow for -1 for end-of-file marker) 1418e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project int m_current; 142ca9cb53ed1119a3fd98fafa0972ffeb56dee1c24Steve Block 143231d4e3152a9c27a73b6ac7badbe6be673aa3ddfSteve Block IdentifierArena* m_arena; 1448e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project 1458e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project JSGlobalData* m_globalData; 1468e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project 1475f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian const HashTable m_keywordTable; 1488e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project }; 1498e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project 1505f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian inline bool Lexer::isWhiteSpace(int ch) 1515f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian { 152967717af5423377c967781471ee106e2bb4e11c8Ben Murdoch return isASCII(ch) ? (ch == ' ' || ch == '\t' || ch == 0xB || ch == 0xC) : (WTF::Unicode::isSeparatorSpace(ch) || ch == 0xFEFF); 1535f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian } 1545f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian 1555f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian inline bool Lexer::isLineTerminator(int ch) 1565f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian { 1575f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian return ch == '\r' || ch == '\n' || (ch & ~1) == 0x2028; 1585f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian } 1595f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian 1605f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian inline unsigned char Lexer::convertHex(int c1, int c2) 1615f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian { 1625f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian return (toASCIIHexValue(c1) << 4) | toASCIIHexValue(c2); 1635f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian } 1645f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian 1655f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian inline UChar Lexer::convertUnicode(int c1, int c2, int c3, int c4) 1665f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian { 1675f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian return (convertHex(c1, c2) << 8) | convertHex(c3, c4); 1685f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian } 1695f1ab04193ad0130ca8204aadaceae083aca9881Feng Qian 1708e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project} // namespace JSC 1718e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project 1728e35f3cfc7fba1d1c829dc557ebad6409cbe16a2The Android Open Source Project#endif // Lexer_h 173