1// Copyright (c) 2013 The Chromium Authors. All rights reserved. 2// Use of this source code is governed by a BSD-style license that can be 3// found in the LICENSE file. 4 5#include "testing/gtest/include/gtest/gtest.h" 6#include "tools/gn/input_file.h" 7#include "tools/gn/token.h" 8#include "tools/gn/tokenizer.h" 9 10namespace { 11 12struct TokenExpectation { 13 Token::Type type; 14 const char* value; 15}; 16 17template<size_t len> 18bool CheckTokenizer(const char* input, const TokenExpectation (&expect)[len]) { 19 InputFile input_file(SourceFile("/test")); 20 input_file.SetContents(input); 21 22 Err err; 23 std::vector<Token> results = Tokenizer::Tokenize(&input_file, &err); 24 25 if (results.size() != len) 26 return false; 27 for (size_t i = 0; i < len; i++) { 28 if (expect[i].type != results[i].type()) 29 return false; 30 if (expect[i].value != results[i].value()) 31 return false; 32 } 33 return true; 34} 35 36} // namespace 37 38TEST(Tokenizer, Empty) { 39 InputFile empty_string_input(SourceFile("/test")); 40 empty_string_input.SetContents(""); 41 42 Err err; 43 std::vector<Token> results = Tokenizer::Tokenize(&empty_string_input, &err); 44 EXPECT_TRUE(results.empty()); 45 46 InputFile whitespace_input(SourceFile("/test")); 47 whitespace_input.SetContents(" \r \n \r\n"); 48 49 results = Tokenizer::Tokenize(&whitespace_input, &err); 50 EXPECT_TRUE(results.empty()); 51} 52 53TEST(Tokenizer, Identifier) { 54 TokenExpectation one_ident[] = { 55 { Token::IDENTIFIER, "foo" } 56 }; 57 EXPECT_TRUE(CheckTokenizer(" foo ", one_ident)); 58} 59 60TEST(Tokenizer, Integer) { 61 TokenExpectation integers[] = { 62 { Token::INTEGER, "123" }, 63 { Token::INTEGER, "-123" } 64 }; 65 EXPECT_TRUE(CheckTokenizer(" 123 -123 ", integers)); 66} 67 68TEST(Tokenizer, IntegerNoSpace) { 69 TokenExpectation integers[] = { 70 { Token::INTEGER, "123" }, 71 { Token::INTEGER, "-123" } 72 }; 73 EXPECT_TRUE(CheckTokenizer(" 123-123 ", integers)); 74} 75 76TEST(Tokenizer, String) { 77 TokenExpectation strings[] = { 78 { Token::STRING, "\"foo\"" }, 79 { Token::STRING, "\"bar\\\"baz\"" }, 80 { Token::STRING, "\"asdf\\\\\"" } 81 }; 82 EXPECT_TRUE(CheckTokenizer(" \"foo\" \"bar\\\"baz\" \"asdf\\\\\" ", 83 strings)); 84} 85 86TEST(Tokenizer, Operator) { 87 TokenExpectation operators[] = { 88 { Token::MINUS, "-" }, 89 { Token::PLUS, "+" }, 90 { Token::EQUAL, "=" }, 91 { Token::PLUS_EQUALS, "+=" }, 92 { Token::MINUS_EQUALS, "-=" }, 93 { Token::NOT_EQUAL, "!=" }, 94 { Token::EQUAL_EQUAL, "==" }, 95 { Token::LESS_THAN, "<" }, 96 { Token::GREATER_THAN, ">" }, 97 { Token::LESS_EQUAL, "<=" }, 98 { Token::GREATER_EQUAL, ">=" }, 99 { Token::BANG, "!" }, 100 { Token::BOOLEAN_OR, "||" }, 101 { Token::BOOLEAN_AND, "&&" }, 102 { Token::DOT, "." }, 103 { Token::COMMA, "," }, 104 }; 105 EXPECT_TRUE(CheckTokenizer("- + = += -= != == < > <= >= ! || && . ,", 106 operators)); 107} 108 109TEST(Tokenizer, Scoper) { 110 TokenExpectation scopers[] = { 111 { Token::LEFT_BRACE, "{" }, 112 { Token::LEFT_BRACKET, "[" }, 113 { Token::RIGHT_BRACKET, "]" }, 114 { Token::RIGHT_BRACE, "}" }, 115 { Token::LEFT_PAREN, "(" }, 116 { Token::RIGHT_PAREN, ")" }, 117 }; 118 EXPECT_TRUE(CheckTokenizer("{[ ]} ()", scopers)); 119} 120 121TEST(Tokenizer, FunctionCall) { 122 TokenExpectation fn[] = { 123 { Token::IDENTIFIER, "fun" }, 124 { Token::LEFT_PAREN, "(" }, 125 { Token::STRING, "\"foo\"" }, 126 { Token::RIGHT_PAREN, ")" }, 127 { Token::LEFT_BRACE, "{" }, 128 { Token::IDENTIFIER, "foo" }, 129 { Token::EQUAL, "=" }, 130 { Token::INTEGER, "12" }, 131 { Token::RIGHT_BRACE, "}" }, 132 }; 133 EXPECT_TRUE(CheckTokenizer("fun(\"foo\") {\nfoo = 12}", fn)); 134} 135 136TEST(Tokenizer, Locations) { 137 InputFile input(SourceFile("/test")); 138 input.SetContents("1 2 \"three\"\n 4"); 139 Err err; 140 std::vector<Token> results = Tokenizer::Tokenize(&input, &err); 141 142 ASSERT_EQ(4u, results.size()); 143 ASSERT_TRUE(results[0].location() == Location(&input, 1, 1, 1)); 144 ASSERT_TRUE(results[1].location() == Location(&input, 1, 3, 3)); 145 ASSERT_TRUE(results[2].location() == Location(&input, 1, 5, 5)); 146 ASSERT_TRUE(results[3].location() == Location(&input, 2, 3, 8)); 147} 148 149TEST(Tokenizer, ByteOffsetOfNthLine) { 150 EXPECT_EQ(0u, Tokenizer::ByteOffsetOfNthLine("foo", 1)); 151 152 // Windows and Posix have different line endings, so check the byte at the 153 // location rather than the offset. 154 char input1[] = "aaa\nxaa\n\nya"; 155 EXPECT_EQ('x', input1[Tokenizer::ByteOffsetOfNthLine(input1, 2)]); 156 EXPECT_EQ('y', input1[Tokenizer::ByteOffsetOfNthLine(input1, 4)]); 157 158 char input2[3]; 159 input2[0] = 'a'; 160 input2[1] = '\n'; // Manually set to avoid Windows double-byte endings. 161 input2[2] = 0; 162 EXPECT_EQ(0u, Tokenizer::ByteOffsetOfNthLine(input2, 1)); 163 EXPECT_EQ(2u, Tokenizer::ByteOffsetOfNthLine(input2, 2)); 164} 165 166TEST(Tokenizer, Comments) { 167 TokenExpectation fn[] = { 168 { Token::LINE_COMMENT, "# Stuff" }, 169 { Token::IDENTIFIER, "fun" }, 170 { Token::LEFT_PAREN, "(" }, 171 { Token::STRING, "\"foo\"" }, 172 { Token::RIGHT_PAREN, ")" }, 173 { Token::LEFT_BRACE, "{" }, 174 { Token::SUFFIX_COMMENT, "# Things" }, 175 { Token::LINE_COMMENT, "#Wee" }, 176 { Token::IDENTIFIER, "foo" }, 177 { Token::EQUAL, "=" }, 178 { Token::INTEGER, "12" }, 179 { Token::SUFFIX_COMMENT, "#Zip" }, 180 { Token::RIGHT_BRACE, "}" }, 181 }; 182 EXPECT_TRUE(CheckTokenizer( 183 "# Stuff\n" 184 "fun(\"foo\") { # Things\n" 185 "#Wee\n" 186 "foo = 12 #Zip\n" 187 "}", 188 fn)); 189} 190 191TEST(Tokenizer, CommentsContinued) { 192 // In the first test, the comments aren't horizontally aligned, so they're 193 // considered separate. In the second test, they are, so "B" is a 194 // continuation of "A" (another SUFFIX comment). 195 TokenExpectation fn1[] = { 196 { Token::IDENTIFIER, "fun" }, 197 { Token::LEFT_PAREN, "(" }, 198 { Token::STRING, "\"foo\"" }, 199 { Token::RIGHT_PAREN, ")" }, 200 { Token::LEFT_BRACE, "{" }, 201 { Token::SUFFIX_COMMENT, "# A" }, 202 { Token::LINE_COMMENT, "# B" }, 203 { Token::RIGHT_BRACE, "}" }, 204 }; 205 EXPECT_TRUE(CheckTokenizer( 206 "fun(\"foo\") { # A\n" 207 " # B\n" 208 "}", 209 fn1)); 210 211 TokenExpectation fn2[] = { 212 { Token::IDENTIFIER, "fun" }, 213 { Token::LEFT_PAREN, "(" }, 214 { Token::STRING, "\"foo\"" }, 215 { Token::RIGHT_PAREN, ")" }, 216 { Token::LEFT_BRACE, "{" }, 217 { Token::SUFFIX_COMMENT, "# A" }, 218 { Token::SUFFIX_COMMENT, "# B" }, 219 { Token::RIGHT_BRACE, "}" }, 220 }; 221 EXPECT_TRUE(CheckTokenizer( 222 "fun(\"foo\") { # A\n" 223 " # B\n" // Note that these are aligned, the \"s move A out. 224 "}", 225 fn2)); 226} 227