1// Copyright (c) 2013 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include <iostream>
6#include <sstream>
7
8#include "testing/gtest/include/gtest/gtest.h"
9#include "tools/gn/input_file.h"
10#include "tools/gn/parser.h"
11#include "tools/gn/tokenizer.h"
12
13namespace {
14
15bool GetTokens(const InputFile* input, std::vector<Token>* result) {
16  result->clear();
17  Err err;
18  *result = Tokenizer::Tokenize(input, &err);
19  return !err.has_error();
20}
21
22void DoParserPrintTest(const char* input, const char* expected) {
23  std::vector<Token> tokens;
24  InputFile input_file(SourceFile("/test"));
25  input_file.SetContents(input);
26  ASSERT_TRUE(GetTokens(&input_file, &tokens));
27
28  Err err;
29  scoped_ptr<ParseNode> result = Parser::Parse(tokens, &err);
30  if (!result)
31    err.PrintToStdout();
32  ASSERT_TRUE(result);
33
34  std::ostringstream collector;
35  result->Print(collector, 0);
36
37  EXPECT_EQ(expected, collector.str());
38}
39
40void DoExpressionPrintTest(const char* input, const char* expected) {
41  std::vector<Token> tokens;
42  InputFile input_file(SourceFile("/test"));
43  input_file.SetContents(input);
44  ASSERT_TRUE(GetTokens(&input_file, &tokens));
45
46  Err err;
47  scoped_ptr<ParseNode> result = Parser::ParseExpression(tokens, &err);
48  ASSERT_TRUE(result);
49
50  std::ostringstream collector;
51  result->Print(collector, 0);
52
53  EXPECT_EQ(expected, collector.str());
54}
55
56// Expects the tokenizer or parser to identify an error at the given line and
57// character.
58void DoParserErrorTest(const char* input, int err_line, int err_char) {
59  InputFile input_file(SourceFile("/test"));
60  input_file.SetContents(input);
61
62  Err err;
63  std::vector<Token> tokens = Tokenizer::Tokenize(&input_file, &err);
64  if (!err.has_error()) {
65    scoped_ptr<ParseNode> result = Parser::Parse(tokens, &err);
66    ASSERT_FALSE(result);
67    ASSERT_TRUE(err.has_error());
68  }
69
70  EXPECT_EQ(err_line, err.location().line_number());
71  EXPECT_EQ(err_char, err.location().char_offset());
72}
73
74// Expects the tokenizer or parser to identify an error at the given line and
75// character.
76void DoExpressionErrorTest(const char* input, int err_line, int err_char) {
77  InputFile input_file(SourceFile("/test"));
78  input_file.SetContents(input);
79
80  Err err;
81  std::vector<Token> tokens = Tokenizer::Tokenize(&input_file, &err);
82  if (!err.has_error()) {
83    scoped_ptr<ParseNode> result = Parser::ParseExpression(tokens, &err);
84    ASSERT_FALSE(result);
85    ASSERT_TRUE(err.has_error());
86  }
87
88  EXPECT_EQ(err_line, err.location().line_number());
89  EXPECT_EQ(err_char, err.location().char_offset());
90}
91
92}  // namespace
93
94TEST(Parser, Literal) {
95  DoExpressionPrintTest("5", "LITERAL(5)\n");
96  DoExpressionPrintTest("\"stuff\"", "LITERAL(\"stuff\")\n");
97}
98
99TEST(Parser, BinaryOp) {
100  // TODO(scottmg): The tokenizer is dumb, and treats "5-1" as two integers,
101  // not a binary operator between two positive integers.
102  DoExpressionPrintTest("5 - 1",
103      "BINARY(-)\n"
104      " LITERAL(5)\n"
105      " LITERAL(1)\n");
106  DoExpressionPrintTest("5+1",
107      "BINARY(+)\n"
108      " LITERAL(5)\n"
109      " LITERAL(1)\n");
110  DoExpressionPrintTest("5 - 1 - 2",
111      "BINARY(-)\n"
112      " BINARY(-)\n"
113      "  LITERAL(5)\n"
114      "  LITERAL(1)\n"
115      " LITERAL(2)\n");
116}
117
118TEST(Parser, FunctionCall) {
119  DoExpressionPrintTest("foo()",
120      "FUNCTION(foo)\n"
121      " LIST\n");
122  DoExpressionPrintTest("blah(1, 2)",
123      "FUNCTION(blah)\n"
124      " LIST\n"
125      "  LITERAL(1)\n"
126      "  LITERAL(2)\n");
127  DoExpressionErrorTest("foo(1, 2,)", 1, 10);
128  DoExpressionErrorTest("foo(1 2)", 1, 7);
129}
130
131TEST(Parser, ParenExpression) {
132  const char* input = "(foo(1)) + (a + (b - c) + d)";
133  const char* expected =
134      "BINARY(+)\n"
135      " FUNCTION(foo)\n"
136      "  LIST\n"
137      "   LITERAL(1)\n"
138      " BINARY(+)\n"
139      "  BINARY(+)\n"
140      "   IDENTIFIER(a)\n"
141      "   BINARY(-)\n"
142      "    IDENTIFIER(b)\n"
143      "    IDENTIFIER(c)\n"
144      "  IDENTIFIER(d)\n";
145  DoExpressionPrintTest(input, expected);
146  DoExpressionErrorTest("(a +", 1, 4);
147}
148
149TEST(Parser, OrderOfOperationsLeftAssociative) {
150  const char* input = "5 - 1 - 2\n";
151  const char* expected =
152      "BINARY(-)\n"
153      " BINARY(-)\n"
154      "  LITERAL(5)\n"
155      "  LITERAL(1)\n"
156      " LITERAL(2)\n";
157  DoExpressionPrintTest(input, expected);
158}
159
160TEST(Parser, OrderOfOperationsEqualityBoolean) {
161  const char* input =
162      "if (a == \"b\" && is_stuff) {\n"
163      "  print(\"hai\")\n"
164      "}\n";
165  const char* expected =
166      "BLOCK\n"
167      " CONDITION\n"
168      "  BINARY(&&)\n"
169      "   BINARY(==)\n"
170      "    IDENTIFIER(a)\n"
171      "    LITERAL(\"b\")\n"
172      "   IDENTIFIER(is_stuff)\n"
173      "  BLOCK\n"
174      "   FUNCTION(print)\n"
175      "    LIST\n"
176      "     LITERAL(\"hai\")\n";
177  DoParserPrintTest(input, expected);
178}
179
180TEST(Parser, UnaryOp) {
181  DoExpressionPrintTest("!foo",
182      "UNARY(!)\n"
183      " IDENTIFIER(foo)\n");
184}
185
186TEST(Parser, List) {
187  DoExpressionPrintTest("[]", "LIST\n");
188  DoExpressionPrintTest("[1,asd,]",
189      "LIST\n"
190      " LITERAL(1)\n"
191      " IDENTIFIER(asd)\n");
192  DoExpressionPrintTest("[1, 2+3 - foo]",
193      "LIST\n"
194      " LITERAL(1)\n"
195      " BINARY(-)\n"
196      "  BINARY(+)\n"
197      "   LITERAL(2)\n"
198      "   LITERAL(3)\n"
199      "  IDENTIFIER(foo)\n");
200  DoExpressionPrintTest("[1,\n2,\n 3,\n  4]",
201      "LIST\n"
202      " LITERAL(1)\n"
203      " LITERAL(2)\n"
204      " LITERAL(3)\n"
205      " LITERAL(4)\n");
206
207  DoExpressionErrorTest("[a, 2+,]", 1, 6);
208  DoExpressionErrorTest("[,]", 1, 2);
209  DoExpressionErrorTest("[a,,]", 1, 4);
210}
211
212TEST(Parser, Assignment) {
213  DoParserPrintTest("a=2",
214                    "BLOCK\n"
215                    " BINARY(=)\n"
216                    "  IDENTIFIER(a)\n"
217                    "  LITERAL(2)\n");
218}
219
220TEST(Parser, Accessor) {
221  // Accessor indexing.
222  DoParserPrintTest("a=b[c+2]",
223                    "BLOCK\n"
224                    " BINARY(=)\n"
225                    "  IDENTIFIER(a)\n"
226                    "  ACCESSOR\n"
227                    "   b\n"  // AccessorNode is a bit weird in that it holds
228                              // a Token, not a ParseNode for the base.
229                    "   BINARY(+)\n"
230                    "    IDENTIFIER(c)\n"
231                    "    LITERAL(2)\n");
232  DoParserErrorTest("a = b[1][0]", 1, 5);
233
234  // Member accessors.
235  DoParserPrintTest("a=b.c+2",
236                    "BLOCK\n"
237                    " BINARY(=)\n"
238                    "  IDENTIFIER(a)\n"
239                    "  BINARY(+)\n"
240                    "   ACCESSOR\n"
241                    "    b\n"
242                    "    IDENTIFIER(c)\n"
243                    "   LITERAL(2)\n");
244  DoParserErrorTest("a = b.c.d", 1, 6);  // Can't nest accessors (currently).
245  DoParserErrorTest("a.b = 5", 1, 1);  // Can't assign to accessors (currently).
246}
247
248TEST(Parser, Condition) {
249  DoParserPrintTest("if(1) { a = 2 }",
250                    "BLOCK\n"
251                    " CONDITION\n"
252                    "  LITERAL(1)\n"
253                    "  BLOCK\n"
254                    "   BINARY(=)\n"
255                    "    IDENTIFIER(a)\n"
256                    "    LITERAL(2)\n");
257
258  DoParserPrintTest("if(1) { a = 2 } else if (0) { a = 3 } else { a = 4 }",
259                    "BLOCK\n"
260                    " CONDITION\n"
261                    "  LITERAL(1)\n"
262                    "  BLOCK\n"
263                    "   BINARY(=)\n"
264                    "    IDENTIFIER(a)\n"
265                    "    LITERAL(2)\n"
266                    "  CONDITION\n"
267                    "   LITERAL(0)\n"
268                    "   BLOCK\n"
269                    "    BINARY(=)\n"
270                    "     IDENTIFIER(a)\n"
271                    "     LITERAL(3)\n"
272                    "   BLOCK\n"
273                    "    BINARY(=)\n"
274                    "     IDENTIFIER(a)\n"
275                    "     LITERAL(4)\n");
276}
277
278TEST(Parser, OnlyCallAndAssignInBody) {
279  DoParserErrorTest("[]", 1, 2);
280  DoParserErrorTest("3 + 4", 1, 5);
281  DoParserErrorTest("6 - 7", 1, 5);
282  DoParserErrorTest("if (1) { 5 } else { print(4) }", 1, 12);
283}
284
285TEST(Parser, NoAssignmentInCondition) {
286  DoParserErrorTest("if (a=2) {}", 1, 5);
287}
288
289TEST(Parser, CompleteFunction) {
290  const char* input =
291      "cc_test(\"foo\") {\n"
292      "  sources = [\n"
293      "    \"foo.cc\",\n"
294      "    \"foo.h\"\n"
295      "  ]\n"
296      "  dependencies = [\n"
297      "    \"base\"\n"
298      "  ]\n"
299      "}\n";
300  const char* expected =
301      "BLOCK\n"
302      " FUNCTION(cc_test)\n"
303      "  LIST\n"
304      "   LITERAL(\"foo\")\n"
305      "  BLOCK\n"
306      "   BINARY(=)\n"
307      "    IDENTIFIER(sources)\n"
308      "    LIST\n"
309      "     LITERAL(\"foo.cc\")\n"
310      "     LITERAL(\"foo.h\")\n"
311      "   BINARY(=)\n"
312      "    IDENTIFIER(dependencies)\n"
313      "    LIST\n"
314      "     LITERAL(\"base\")\n";
315  DoParserPrintTest(input, expected);
316}
317
318TEST(Parser, FunctionWithConditional) {
319  const char* input =
320      "cc_test(\"foo\") {\n"
321      "  sources = [\"foo.cc\"]\n"
322      "  if (OS == \"mac\") {\n"
323      "    sources += \"bar.cc\"\n"
324      "  } else if (OS == \"win\") {\n"
325      "    sources -= [\"asd.cc\", \"foo.cc\"]\n"
326      "  } else {\n"
327      "    dependencies += [\"bar.cc\"]\n"
328      "  }\n"
329      "}\n";
330  const char* expected =
331      "BLOCK\n"
332      " FUNCTION(cc_test)\n"
333      "  LIST\n"
334      "   LITERAL(\"foo\")\n"
335      "  BLOCK\n"
336      "   BINARY(=)\n"
337      "    IDENTIFIER(sources)\n"
338      "    LIST\n"
339      "     LITERAL(\"foo.cc\")\n"
340      "   CONDITION\n"
341      "    BINARY(==)\n"
342      "     IDENTIFIER(OS)\n"
343      "     LITERAL(\"mac\")\n"
344      "    BLOCK\n"
345      "     BINARY(+=)\n"
346      "      IDENTIFIER(sources)\n"
347      "      LITERAL(\"bar.cc\")\n"
348      "    CONDITION\n"
349      "     BINARY(==)\n"
350      "      IDENTIFIER(OS)\n"
351      "      LITERAL(\"win\")\n"
352      "     BLOCK\n"
353      "      BINARY(-=)\n"
354      "       IDENTIFIER(sources)\n"
355      "       LIST\n"
356      "        LITERAL(\"asd.cc\")\n"
357      "        LITERAL(\"foo.cc\")\n"
358      "     BLOCK\n"
359      "      BINARY(+=)\n"
360      "       IDENTIFIER(dependencies)\n"
361      "       LIST\n"
362      "        LITERAL(\"bar.cc\")\n";
363  DoParserPrintTest(input, expected);
364}
365
366TEST(Parser, NestedBlocks) {
367  const char* input = "{cc_test(\"foo\") {{foo=1}\n{}}}";
368  const char* expected =
369      "BLOCK\n"
370      " BLOCK\n"
371      "  FUNCTION(cc_test)\n"
372      "   LIST\n"
373      "    LITERAL(\"foo\")\n"
374      "   BLOCK\n"
375      "    BLOCK\n"
376      "     BINARY(=)\n"
377      "      IDENTIFIER(foo)\n"
378      "      LITERAL(1)\n"
379      "    BLOCK\n";
380  DoParserPrintTest(input, expected);
381  const char* input_with_newline = "{cc_test(\"foo\") {{foo=1}\n{}}}";
382  DoParserPrintTest(input_with_newline, expected);
383}
384
385TEST(Parser, UnterminatedBlock) {
386  DoParserErrorTest("stuff() {", 1, 9);
387}
388
389TEST(Parser, BadlyTerminatedNumber) {
390  DoParserErrorTest("1234z", 1, 5);
391}
392
393TEST(Parser, NewlinesInUnusualPlaces) {
394  DoParserPrintTest(
395      "if\n"
396      "(\n"
397      "a\n"
398      ")\n"
399      "{\n"
400      "}\n",
401      "BLOCK\n"
402      " CONDITION\n"
403      "  IDENTIFIER(a)\n"
404      "  BLOCK\n");
405}
406
407TEST(Parser, NewlinesInUnusualPlaces2) {
408  DoParserPrintTest(
409      "a\n=\n2\n",
410      "BLOCK\n"
411      " BINARY(=)\n"
412      "  IDENTIFIER(a)\n"
413      "  LITERAL(2)\n");
414  DoParserPrintTest(
415      "x =\ny if\n(1\n) {}",
416      "BLOCK\n"
417      " BINARY(=)\n"
418      "  IDENTIFIER(x)\n"
419      "  IDENTIFIER(y)\n"
420      " CONDITION\n"
421      "  LITERAL(1)\n"
422      "  BLOCK\n");
423  DoParserPrintTest(
424      "x = 3\n+2",
425      "BLOCK\n"
426      " BINARY(=)\n"
427      "  IDENTIFIER(x)\n"
428      "  BINARY(+)\n"
429      "   LITERAL(3)\n"
430      "   LITERAL(2)\n"
431      );
432}
433
434TEST(Parser, NewlineBeforeSubscript) {
435  const char* input = "a = b[1]";
436  const char* input_with_newline = "a = b\n[1]";
437  const char* expected =
438    "BLOCK\n"
439    " BINARY(=)\n"
440    "  IDENTIFIER(a)\n"
441    "  ACCESSOR\n"
442    "   b\n"
443    "   LITERAL(1)\n";
444  DoParserPrintTest(
445      input,
446      expected);
447  DoParserPrintTest(
448      input_with_newline,
449      expected);
450}
451
452TEST(Parser, SequenceOfExpressions) {
453  DoParserPrintTest(
454      "a = 1 b = 2",
455      "BLOCK\n"
456      " BINARY(=)\n"
457      "  IDENTIFIER(a)\n"
458      "  LITERAL(1)\n"
459      " BINARY(=)\n"
460      "  IDENTIFIER(b)\n"
461      "  LITERAL(2)\n");
462}
463
464TEST(Parser, BlockAfterFunction) {
465  const char* input = "func(\"stuff\") {\n}";
466  // TODO(scottmg): Do we really want these to mean different things?
467  const char* input_with_newline = "func(\"stuff\")\n{\n}";
468  const char* expected =
469    "BLOCK\n"
470    " FUNCTION(func)\n"
471    "  LIST\n"
472    "   LITERAL(\"stuff\")\n"
473    "  BLOCK\n";
474  DoParserPrintTest(input, expected);
475  DoParserPrintTest(input_with_newline, expected);
476}
477
478TEST(Parser, LongExpression) {
479  const char* input = "a = b + c && d || e";
480  const char* expected =
481    "BLOCK\n"
482    " BINARY(=)\n"
483    "  IDENTIFIER(a)\n"
484    "  BINARY(||)\n"
485    "   BINARY(&&)\n"
486    "    BINARY(+)\n"
487    "     IDENTIFIER(b)\n"
488    "     IDENTIFIER(c)\n"
489    "    IDENTIFIER(d)\n"
490    "   IDENTIFIER(e)\n";
491  DoParserPrintTest(input, expected);
492}
493
494TEST(Parser, CommentsStandalone) {
495  const char* input =
496    "# Toplevel comment.\n"
497    "\n"
498    "executable(\"wee\") {}\n";
499  const char* expected =
500    "BLOCK\n"
501    " BLOCK_COMMENT(# Toplevel comment.)\n"
502    " FUNCTION(executable)\n"
503    "  LIST\n"
504    "   LITERAL(\"wee\")\n"
505    "  BLOCK\n";
506  DoParserPrintTest(input, expected);
507}
508
509TEST(Parser, CommentsStandaloneEof) {
510  const char* input =
511    "executable(\"wee\") {}\n"
512    "# EOF comment.\n";
513  const char* expected =
514    "BLOCK\n"
515    " +AFTER_COMMENT(\"# EOF comment.\")\n"
516    " FUNCTION(executable)\n"
517    "  LIST\n"
518    "   LITERAL(\"wee\")\n"
519    "  BLOCK\n";
520  DoParserPrintTest(input, expected);
521}
522
523TEST(Parser, CommentsLineAttached) {
524  const char* input =
525    "executable(\"wee\") {\n"
526    "  # Some sources.\n"
527    "  sources = [\n"
528    "    \"stuff.cc\",\n"
529    "    \"things.cc\",\n"
530    "    # This file is special or something.\n"
531    "    \"another.cc\",\n"
532    "  ]\n"
533    "}\n";
534  const char* expected =
535    "BLOCK\n"
536    " FUNCTION(executable)\n"
537    "  LIST\n"
538    "   LITERAL(\"wee\")\n"
539    "  BLOCK\n"
540    "   BINARY(=)\n"
541    "    +BEFORE_COMMENT(\"# Some sources.\")\n"
542    "    IDENTIFIER(sources)\n"
543    "    LIST\n"
544    "     LITERAL(\"stuff.cc\")\n"
545    "     LITERAL(\"things.cc\")\n"
546    "     LITERAL(\"another.cc\")\n"
547    "      +BEFORE_COMMENT(\"# This file is special or something.\")\n";
548  DoParserPrintTest(input, expected);
549}
550
551TEST(Parser, CommentsSuffix) {
552  const char* input =
553    "executable(\"wee\") { # This is some stuff.\n"
554    "sources = [ \"a.cc\" # And another comment here.\n"
555    "] }";
556  const char* expected =
557    "BLOCK\n"
558    " FUNCTION(executable)\n"
559    "  LIST\n"
560    "   LITERAL(\"wee\")\n"
561    "    +SUFFIX_COMMENT(\"# This is some stuff.\")\n"
562    "  BLOCK\n"
563    "   BINARY(=)\n"
564    "    IDENTIFIER(sources)\n"
565    "    LIST\n"
566    "     LITERAL(\"a.cc\")\n"
567    "      +SUFFIX_COMMENT(\"# And another comment here.\")\n";
568  DoParserPrintTest(input, expected);
569}
570
571TEST(Parser, CommentsSuffixDifferentLine) {
572  const char* input =
573    "executable(\"wee\") {\n"
574    "  sources = [ \"a\",\n"
575    "      \"b\" ] # Comment\n"
576    "}\n";
577  const char* expected =
578    "BLOCK\n"
579    " FUNCTION(executable)\n"
580    "  LIST\n"
581    "   LITERAL(\"wee\")\n"
582    "  BLOCK\n"
583    "   BINARY(=)\n"
584    "    IDENTIFIER(sources)\n"
585    "    LIST\n"
586    "     LITERAL(\"a\")\n"
587    "     LITERAL(\"b\")\n"
588    "      +SUFFIX_COMMENT(\"# Comment\")\n";
589  DoParserPrintTest(input, expected);
590}
591
592TEST(Parser, CommentsSuffixMultiple) {
593  const char* input =
594    "executable(\"wee\") {\n"
595    "  sources = [\n"
596    "    \"a\",  # This is a comment,\n"
597    "          # and some more,\n"  // Note that this is aligned with above.
598    "          # then the end.\n"
599    "  ]\n"
600    "}\n";
601  const char* expected =
602    "BLOCK\n"
603    " FUNCTION(executable)\n"
604    "  LIST\n"
605    "   LITERAL(\"wee\")\n"
606    "  BLOCK\n"
607    "   BINARY(=)\n"
608    "    IDENTIFIER(sources)\n"
609    "    LIST\n"
610    "     LITERAL(\"a\")\n"
611    "      +SUFFIX_COMMENT(\"# This is a comment,\")\n"
612    "      +SUFFIX_COMMENT(\"# and some more,\")\n"
613    "      +SUFFIX_COMMENT(\"# then the end.\")\n";
614  DoParserPrintTest(input, expected);
615}
616
617TEST(Parser, CommentsConnectedInList) {
618  const char* input =
619    "defines = [\n"
620    "\n"
621    "  # Connected comment.\n"
622    "  \"WEE\",\n"
623    "  \"BLORPY\",\n"
624    "]\n";
625  const char* expected =
626    "BLOCK\n"
627    " BINARY(=)\n"
628    "  IDENTIFIER(defines)\n"
629    "  LIST\n"
630    "   LITERAL(\"WEE\")\n"
631    "    +BEFORE_COMMENT(\"# Connected comment.\")\n"
632    "   LITERAL(\"BLORPY\")\n";
633  DoParserPrintTest(input, expected);
634}
635
636TEST(Parser, HangingIf) {
637  DoParserErrorTest("if", 1, 1);
638}
639
640TEST(Parser, NegatingList) {
641  DoParserErrorTest("executable(\"wee\") { sources =- [ \"foo.cc\" ] }", 1, 30);
642}
643