1// Copyright (c) 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "testing/gtest/include/gtest/gtest.h"
6#include "tools/gn/input_file.h"
7#include "tools/gn/token.h"
8#include "tools/gn/tokenizer.h"
9
10namespace {
11
12struct TokenExpectation {
13  Token::Type type;
14  const char* value;
15};
16
17template<size_t len>
18bool CheckTokenizer(const char* input, const TokenExpectation (&expect)[len]) {
19  InputFile input_file(SourceFile("/test"));
20  input_file.SetContents(input);
21
22  Err err;
23  std::vector<Token> results = Tokenizer::Tokenize(&input_file, &err);
24
25  if (results.size() != len)
26    return false;
27  for (size_t i = 0; i < len; i++) {
28    if (expect[i].type != results[i].type())
29      return false;
30    if (expect[i].value != results[i].value())
31      return false;
32  }
33  return true;
34}
35
36}  // namespace
37
38TEST(Tokenizer, Empty) {
39  InputFile empty_string_input(SourceFile("/test"));
40  empty_string_input.SetContents("");
41
42  Err err;
43  std::vector<Token> results = Tokenizer::Tokenize(&empty_string_input, &err);
44  EXPECT_TRUE(results.empty());
45
46  InputFile whitespace_input(SourceFile("/test"));
47  whitespace_input.SetContents("  \r \n \r\n");
48
49  results = Tokenizer::Tokenize(&whitespace_input, &err);
50  EXPECT_TRUE(results.empty());
51}
52
53TEST(Tokenizer, Identifier) {
54  TokenExpectation one_ident[] = {
55    { Token::IDENTIFIER, "foo" }
56  };
57  EXPECT_TRUE(CheckTokenizer("  foo ", one_ident));
58}
59
60TEST(Tokenizer, Integer) {
61  TokenExpectation integers[] = {
62    { Token::INTEGER, "123" },
63    { Token::INTEGER, "-123" }
64  };
65  EXPECT_TRUE(CheckTokenizer("  123 -123 ", integers));
66}
67
68TEST(Tokenizer, IntegerNoSpace) {
69  TokenExpectation integers[] = {
70    { Token::INTEGER, "123" },
71    { Token::INTEGER, "-123" }
72  };
73  EXPECT_TRUE(CheckTokenizer("  123-123 ", integers));
74}
75
76TEST(Tokenizer, String) {
77  TokenExpectation strings[] = {
78    { Token::STRING, "\"foo\"" },
79    { Token::STRING, "\"bar\\\"baz\"" },
80    { Token::STRING, "\"asdf\\\\\"" }
81  };
82  EXPECT_TRUE(CheckTokenizer("  \"foo\" \"bar\\\"baz\" \"asdf\\\\\" ",
83              strings));
84}
85
86TEST(Tokenizer, Operator) {
87  TokenExpectation operators[] = {
88    { Token::MINUS, "-" },
89    { Token::PLUS, "+" },
90    { Token::EQUAL, "=" },
91    { Token::PLUS_EQUALS, "+=" },
92    { Token::MINUS_EQUALS, "-=" },
93    { Token::NOT_EQUAL, "!=" },
94    { Token::EQUAL_EQUAL, "==" },
95    { Token::LESS_THAN, "<" },
96    { Token::GREATER_THAN, ">" },
97    { Token::LESS_EQUAL, "<=" },
98    { Token::GREATER_EQUAL, ">=" },
99    { Token::BANG, "!" },
100    { Token::BOOLEAN_OR, "||" },
101    { Token::BOOLEAN_AND, "&&" },
102    { Token::DOT, "." },
103    { Token::COMMA, "," },
104  };
105  EXPECT_TRUE(CheckTokenizer("- + = += -= != ==  < > <= >= ! || && . ,",
106              operators));
107}
108
109TEST(Tokenizer, Scoper) {
110  TokenExpectation scopers[] = {
111    { Token::LEFT_BRACE, "{" },
112    { Token::LEFT_BRACKET, "[" },
113    { Token::RIGHT_BRACKET, "]" },
114    { Token::RIGHT_BRACE, "}" },
115    { Token::LEFT_PAREN, "(" },
116    { Token::RIGHT_PAREN, ")" },
117  };
118  EXPECT_TRUE(CheckTokenizer("{[ ]} ()", scopers));
119}
120
121TEST(Tokenizer, FunctionCall) {
122  TokenExpectation fn[] = {
123    { Token::IDENTIFIER, "fun" },
124    { Token::LEFT_PAREN, "(" },
125    { Token::STRING, "\"foo\"" },
126    { Token::RIGHT_PAREN, ")" },
127    { Token::LEFT_BRACE, "{" },
128    { Token::IDENTIFIER, "foo" },
129    { Token::EQUAL, "=" },
130    { Token::INTEGER, "12" },
131    { Token::RIGHT_BRACE, "}" },
132  };
133  EXPECT_TRUE(CheckTokenizer("fun(\"foo\") {\nfoo = 12}", fn));
134}
135
136TEST(Tokenizer, Locations) {
137  InputFile input(SourceFile("/test"));
138  input.SetContents("1 2 \"three\"\n  4");
139  Err err;
140  std::vector<Token> results = Tokenizer::Tokenize(&input, &err);
141
142  ASSERT_EQ(4u, results.size());
143  ASSERT_TRUE(results[0].location() == Location(&input, 1, 1, 1));
144  ASSERT_TRUE(results[1].location() == Location(&input, 1, 3, 3));
145  ASSERT_TRUE(results[2].location() == Location(&input, 1, 5, 5));
146  ASSERT_TRUE(results[3].location() == Location(&input, 2, 3, 8));
147}
148
149TEST(Tokenizer, ByteOffsetOfNthLine) {
150  EXPECT_EQ(0u, Tokenizer::ByteOffsetOfNthLine("foo", 1));
151
152  // Windows and Posix have different line endings, so check the byte at the
153  // location rather than the offset.
154  char input1[] = "aaa\nxaa\n\nya";
155  EXPECT_EQ('x', input1[Tokenizer::ByteOffsetOfNthLine(input1, 2)]);
156  EXPECT_EQ('y', input1[Tokenizer::ByteOffsetOfNthLine(input1, 4)]);
157
158  char input2[3];
159  input2[0] = 'a';
160  input2[1] = '\n';  // Manually set to avoid Windows double-byte endings.
161  input2[2] = 0;
162  EXPECT_EQ(0u, Tokenizer::ByteOffsetOfNthLine(input2, 1));
163  EXPECT_EQ(2u, Tokenizer::ByteOffsetOfNthLine(input2, 2));
164}
165
166TEST(Tokenizer, Comments) {
167  TokenExpectation fn[] = {
168    { Token::LINE_COMMENT, "# Stuff" },
169    { Token::IDENTIFIER, "fun" },
170    { Token::LEFT_PAREN, "(" },
171    { Token::STRING, "\"foo\"" },
172    { Token::RIGHT_PAREN, ")" },
173    { Token::LEFT_BRACE, "{" },
174    { Token::SUFFIX_COMMENT, "# Things" },
175    { Token::LINE_COMMENT, "#Wee" },
176    { Token::IDENTIFIER, "foo" },
177    { Token::EQUAL, "=" },
178    { Token::INTEGER, "12" },
179    { Token::SUFFIX_COMMENT, "#Zip" },
180    { Token::RIGHT_BRACE, "}" },
181  };
182  EXPECT_TRUE(CheckTokenizer(
183      "# Stuff\n"
184      "fun(\"foo\") {  # Things\n"
185      "#Wee\n"
186      "foo = 12 #Zip\n"
187      "}",
188      fn));
189}
190
191TEST(Tokenizer, CommentsContinued) {
192  // In the first test, the comments aren't horizontally aligned, so they're
193  // considered separate. In the second test, they are, so "B" is a
194  // continuation of "A" (another SUFFIX comment).
195  TokenExpectation fn1[] = {
196    { Token::IDENTIFIER, "fun" },
197    { Token::LEFT_PAREN, "(" },
198    { Token::STRING, "\"foo\"" },
199    { Token::RIGHT_PAREN, ")" },
200    { Token::LEFT_BRACE, "{" },
201    { Token::SUFFIX_COMMENT, "# A" },
202    { Token::LINE_COMMENT, "# B" },
203    { Token::RIGHT_BRACE, "}" },
204  };
205  EXPECT_TRUE(CheckTokenizer(
206      "fun(\"foo\") {  # A\n"
207      "  # B\n"
208      "}",
209      fn1));
210
211  TokenExpectation fn2[] = {
212    { Token::IDENTIFIER, "fun" },
213    { Token::LEFT_PAREN, "(" },
214    { Token::STRING, "\"foo\"" },
215    { Token::RIGHT_PAREN, ")" },
216    { Token::LEFT_BRACE, "{" },
217    { Token::SUFFIX_COMMENT, "# A" },
218    { Token::SUFFIX_COMMENT, "# B" },
219    { Token::RIGHT_BRACE, "}" },
220  };
221  EXPECT_TRUE(CheckTokenizer(
222      "fun(\"foo\") {  # A\n"
223      "              # B\n"  // Note that these are aligned, the \"s move A out.
224      "}",
225      fn2));
226}
227