UnwrappedLineParser.cpp revision 3f4535e44965f46cf603146141d2d27a1cef1f7f 
1//===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9///
10/// \file
11/// \brief This file contains the implementation of the UnwrappedLineParser,
12/// which turns a stream of tokens into UnwrappedLines.
13///
14//===----------------------------------------------------------------------===//
15
16#define DEBUG_TYPE "format-parser"
17
18#include "UnwrappedLineParser.h"
19#include "clang/Basic/Diagnostic.h"
20#include "llvm/Support/Debug.h"
21
22namespace clang {
23namespace format {
24
25class ScopedDeclarationState {
26public:
27  ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
28                         bool MustBeDeclaration)
29      : Line(Line), Stack(Stack) {
30    Line.MustBeDeclaration = MustBeDeclaration;
31    Stack.push_back(MustBeDeclaration);
32  }
33  ~ScopedDeclarationState() {
34    Stack.pop_back();
35    if (!Stack.empty())
36      Line.MustBeDeclaration = Stack.back();
37    else
38      Line.MustBeDeclaration = true;
39  }
40private:
41  UnwrappedLine &Line;
42  std::vector<bool> &Stack;
43};
44
45class ScopedMacroState : public FormatTokenSource {
46public:
47  ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
48                   FormatToken &ResetToken)
49      : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
50        PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource) {
51    TokenSource = this;
52    Line.Level = 0;
53    Line.InPPDirective = true;
54  }
55
56  ~ScopedMacroState() {
57    TokenSource = PreviousTokenSource;
58    ResetToken = Token;
59    Line.InPPDirective = false;
60    Line.Level = PreviousLineLevel;
61  }
62
63  virtual FormatToken getNextToken() {
64    // The \c UnwrappedLineParser guards against this by never calling
65    // \c getNextToken() after it has encountered the first eof token.
66    assert(!eof());
67    Token = PreviousTokenSource->getNextToken();
68    if (eof())
69      return createEOF();
70    return Token;
71  }
72
73private:
74  bool eof() {
75    return Token.NewlinesBefore > 0 && Token.HasUnescapedNewline;
76  }
77
78  FormatToken createEOF() {
79    FormatToken FormatTok;
80    FormatTok.Tok.startToken();
81    FormatTok.Tok.setKind(tok::eof);
82    return FormatTok;
83  }
84
85  UnwrappedLine &Line;
86  FormatTokenSource *&TokenSource;
87  FormatToken &ResetToken;
88  unsigned PreviousLineLevel;
89  FormatTokenSource *PreviousTokenSource;
90
91  FormatToken Token;
92};
93
94class ScopedLineState {
95public:
96  ScopedLineState(UnwrappedLineParser &Parser,
97                  bool SwitchToPreprocessorLines = false)
98      : Parser(Parser), SwitchToPreprocessorLines(SwitchToPreprocessorLines) {
99    if (SwitchToPreprocessorLines)
100      Parser.CurrentLines = &Parser.PreprocessorDirectives;
101    PreBlockLine = Parser.Line.take();
102    Parser.Line.reset(new UnwrappedLine());
103    Parser.Line->Level = PreBlockLine->Level;
104    Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
105  }
106
107  ~ScopedLineState() {
108    if (!Parser.Line->Tokens.empty()) {
109      Parser.addUnwrappedLine();
110    }
111    assert(Parser.Line->Tokens.empty());
112    Parser.Line.reset(PreBlockLine);
113    Parser.MustBreakBeforeNextToken = true;
114    if (SwitchToPreprocessorLines)
115      Parser.CurrentLines = &Parser.Lines;
116  }
117
118private:
119  UnwrappedLineParser &Parser;
120  const bool SwitchToPreprocessorLines;
121
122  UnwrappedLine *PreBlockLine;
123};
124
125UnwrappedLineParser::UnwrappedLineParser(
126    clang::DiagnosticsEngine &Diag, const FormatStyle &Style,
127    FormatTokenSource &Tokens, UnwrappedLineConsumer &Callback)
128    : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
129      CurrentLines(&Lines), Diag(Diag), Style(Style), Tokens(&Tokens),
130      Callback(Callback) {}
131
132bool UnwrappedLineParser::parse() {
133  DEBUG(llvm::dbgs() << "----\n");
134  readToken();
135  bool Error = parseFile();
136  for (std::vector<UnwrappedLine>::iterator I = Lines.begin(),
137                                            E = Lines.end();
138       I != E; ++I) {
139    Callback.consumeUnwrappedLine(*I);
140  }
141  return Error;
142}
143
144bool UnwrappedLineParser::parseFile() {
145  ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
146                                          /*MustBeDeclaration=*/ true);
147  bool Error = parseLevel(/*HasOpeningBrace=*/false);
148  // Make sure to format the remaining tokens.
149  flushComments(true);
150  addUnwrappedLine();
151  return Error;
152}
153
154bool UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
155  bool Error = false;
156  do {
157    switch (FormatTok.Tok.getKind()) {
158    case tok::comment:
159      nextToken();
160      addUnwrappedLine();
161      break;
162    case tok::l_brace:
163      // FIXME: Add parameter whether this can happen - if this happens, we must
164      // be in a non-declaration context.
165      Error |= parseBlock(/*MustBeDeclaration=*/ false);
166      addUnwrappedLine();
167      break;
168    case tok::r_brace:
169      if (HasOpeningBrace) {
170        return false;
171      } else {
172        Diag.Report(FormatTok.Tok.getLocation(),
173                    Diag.getCustomDiagID(clang::DiagnosticsEngine::Error,
174                                         "unexpected '}'"));
175        Error = true;
176        nextToken();
177        addUnwrappedLine();
178      }
179      break;
180    default:
181      parseStructuralElement();
182      break;
183    }
184  } while (!eof());
185  return Error;
186}
187
188bool UnwrappedLineParser::parseBlock(bool MustBeDeclaration,
189                                     unsigned AddLevels) {
190  assert(FormatTok.Tok.is(tok::l_brace) && "'{' expected");
191  nextToken();
192
193  addUnwrappedLine();
194
195  ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
196                                          MustBeDeclaration);
197  Line->Level += AddLevels;
198  parseLevel(/*HasOpeningBrace=*/true);
199
200  if (!FormatTok.Tok.is(tok::r_brace)) {
201    Line->Level -= AddLevels;
202    return true;
203  }
204
205  nextToken();  // Munch the closing brace.
206  Line->Level -= AddLevels;
207  return false;
208}
209
210void UnwrappedLineParser::parsePPDirective() {
211  assert(FormatTok.Tok.is(tok::hash) && "'#' expected");
212  ScopedMacroState MacroState(*Line, Tokens, FormatTok);
213  nextToken();
214
215  if (FormatTok.Tok.getIdentifierInfo() == NULL) {
216    parsePPUnknown();
217    return;
218  }
219
220  switch (FormatTok.Tok.getIdentifierInfo()->getPPKeywordID()) {
221  case tok::pp_define:
222    parsePPDefine();
223    break;
224  default:
225    parsePPUnknown();
226    break;
227  }
228}
229
230void UnwrappedLineParser::parsePPDefine() {
231  nextToken();
232
233  if (FormatTok.Tok.getKind() != tok::identifier) {
234    parsePPUnknown();
235    return;
236  }
237  nextToken();
238  if (FormatTok.Tok.getKind() == tok::l_paren &&
239      FormatTok.WhiteSpaceLength == 0) {
240    parseParens();
241  }
242  addUnwrappedLine();
243  Line->Level = 1;
244
245  // Errors during a preprocessor directive can only affect the layout of the
246  // preprocessor directive, and thus we ignore them. An alternative approach
247  // would be to use the same approach we use on the file level (no
248  // re-indentation if there was a structural error) within the macro
249  // definition.
250  parseFile();
251}
252
253void UnwrappedLineParser::parsePPUnknown() {
254  do {
255    nextToken();
256  } while (!eof());
257  addUnwrappedLine();
258}
259
260void UnwrappedLineParser::parseStructuralElement() {
261  assert(!FormatTok.Tok.is(tok::l_brace));
262  int TokenNumber = 0;
263  switch (FormatTok.Tok.getKind()) {
264  case tok::at:
265    nextToken();
266    if (FormatTok.Tok.is(tok::l_brace)) {
267      parseBracedList();
268      break;
269    }
270    switch (FormatTok.Tok.getObjCKeywordID()) {
271    case tok::objc_public:
272    case tok::objc_protected:
273    case tok::objc_package:
274    case tok::objc_private:
275      return parseAccessSpecifier();
276    case tok::objc_interface:
277    case tok::objc_implementation:
278      return parseObjCInterfaceOrImplementation();
279    case tok::objc_protocol:
280      return parseObjCProtocol();
281    case tok::objc_end:
282      return; // Handled by the caller.
283    case tok::objc_optional:
284    case tok::objc_required:
285      nextToken();
286      addUnwrappedLine();
287      return;
288    default:
289      break;
290    }
291    break;
292  case tok::kw_namespace:
293    parseNamespace();
294    return;
295  case tok::kw_inline:
296    nextToken();
297    TokenNumber++;
298    if (FormatTok.Tok.is(tok::kw_namespace)) {
299      parseNamespace();
300      return;
301    }
302    break;
303  case tok::kw_public:
304  case tok::kw_protected:
305  case tok::kw_private:
306    parseAccessSpecifier();
307    return;
308  case tok::kw_if:
309    parseIfThenElse();
310    return;
311  case tok::kw_for:
312  case tok::kw_while:
313    parseForOrWhileLoop();
314    return;
315  case tok::kw_do:
316    parseDoWhile();
317    return;
318  case tok::kw_switch:
319    parseSwitch();
320    return;
321  case tok::kw_default:
322    nextToken();
323    parseLabel();
324    return;
325  case tok::kw_case:
326    parseCaseLabel();
327    return;
328  case tok::kw_return:
329    parseReturn();
330    return;
331  case tok::kw_extern:
332    nextToken();
333    if (FormatTok.Tok.is(tok::string_literal)) {
334      nextToken();
335      if (FormatTok.Tok.is(tok::l_brace)) {
336        parseBlock(/*MustBeDeclaration=*/ true, 0);
337        addUnwrappedLine();
338        return;
339      }
340    }
341    // In all other cases, parse the declaration.
342    break;
343  default:
344    break;
345  }
346  do {
347    ++TokenNumber;
348    switch (FormatTok.Tok.getKind()) {
349    case tok::at:
350      nextToken();
351      if (FormatTok.Tok.is(tok::l_brace))
352        parseBracedList();
353      break;
354    case tok::kw_enum:
355      parseEnum();
356      break;
357    case tok::kw_struct:
358    case tok::kw_union:
359    case tok::kw_class:
360      parseRecord();
361      // A record declaration or definition is always the start of a structural
362      // element.
363      break;
364    case tok::semi:
365      nextToken();
366      addUnwrappedLine();
367      return;
368    case tok::r_brace:
369      addUnwrappedLine();
370      return;
371    case tok::l_paren:
372      parseParens();
373      break;
374    case tok::l_brace:
375      // A block outside of parentheses must be the last part of a
376      // structural element.
377      // FIXME: Figure out cases where this is not true, and add projections for
378      // them (the one we know is missing are lambdas).
379      parseBlock(/*MustBeDeclaration=*/ false);
380      addUnwrappedLine();
381      return;
382    case tok::identifier:
383      nextToken();
384      if (TokenNumber == 1 && FormatTok.Tok.is(tok::colon)) {
385        parseLabel();
386        return;
387      }
388      break;
389    case tok::equal:
390      nextToken();
391      if (FormatTok.Tok.is(tok::l_brace)) {
392        parseBracedList();
393      }
394      break;
395    default:
396      nextToken();
397      break;
398    }
399  } while (!eof());
400}
401
402void UnwrappedLineParser::parseBracedList() {
403  nextToken();
404
405  do {
406    switch (FormatTok.Tok.getKind()) {
407    case tok::l_brace:
408      parseBracedList();
409      break;
410    case tok::r_brace:
411      nextToken();
412      return;
413    default:
414      nextToken();
415      break;
416    }
417  } while (!eof());
418}
419
420void UnwrappedLineParser::parseReturn() {
421  nextToken();
422
423  do {
424    switch (FormatTok.Tok.getKind()) {
425    case tok::l_brace:
426      parseBracedList();
427      break;
428    case tok::l_paren:
429      parseParens();
430      break;
431    case tok::r_brace:
432      // Assume missing ';'.
433      addUnwrappedLine();
434      return;
435    case tok::semi:
436      nextToken();
437      addUnwrappedLine();
438      return;
439    default:
440      nextToken();
441      break;
442    }
443  } while (!eof());
444}
445
446void UnwrappedLineParser::parseParens() {
447  assert(FormatTok.Tok.is(tok::l_paren) && "'(' expected.");
448  nextToken();
449  do {
450    switch (FormatTok.Tok.getKind()) {
451    case tok::l_paren:
452      parseParens();
453      break;
454    case tok::r_paren:
455      nextToken();
456      return;
457    case tok::l_brace: {
458      nextToken();
459      ScopedLineState LineState(*this);
460      ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
461                                              /*MustBeDeclaration=*/ false);
462      Line->Level += 1;
463      parseLevel(/*HasOpeningBrace=*/ true);
464      Line->Level -= 1;
465      break;
466    }
467    case tok::at:
468      nextToken();
469      if (FormatTok.Tok.is(tok::l_brace))
470        parseBracedList();
471      break;
472    default:
473      nextToken();
474      break;
475    }
476  } while (!eof());
477}
478
479void UnwrappedLineParser::parseIfThenElse() {
480  assert(FormatTok.Tok.is(tok::kw_if) && "'if' expected");
481  nextToken();
482  if (FormatTok.Tok.is(tok::l_paren))
483    parseParens();
484  bool NeedsUnwrappedLine = false;
485  if (FormatTok.Tok.is(tok::l_brace)) {
486    parseBlock(/*MustBeDeclaration=*/ false);
487    NeedsUnwrappedLine = true;
488  } else {
489    addUnwrappedLine();
490    ++Line->Level;
491    parseStructuralElement();
492    --Line->Level;
493  }
494  if (FormatTok.Tok.is(tok::kw_else)) {
495    nextToken();
496    if (FormatTok.Tok.is(tok::l_brace)) {
497      parseBlock(/*MustBeDeclaration=*/ false);
498      addUnwrappedLine();
499    } else if (FormatTok.Tok.is(tok::kw_if)) {
500      parseIfThenElse();
501    } else {
502      addUnwrappedLine();
503      ++Line->Level;
504      parseStructuralElement();
505      --Line->Level;
506    }
507  } else if (NeedsUnwrappedLine) {
508    addUnwrappedLine();
509  }
510}
511
512void UnwrappedLineParser::parseNamespace() {
513  assert(FormatTok.Tok.is(tok::kw_namespace) && "'namespace' expected");
514  nextToken();
515  if (FormatTok.Tok.is(tok::identifier))
516    nextToken();
517  if (FormatTok.Tok.is(tok::l_brace)) {
518    parseBlock(/*MustBeDeclaration=*/ true, 0);
519    // Munch the semicolon after a namespace. This is more common than one would
520    // think. Puttin the semicolon into its own line is very ugly.
521    if (FormatTok.Tok.is(tok::semi))
522      nextToken();
523    addUnwrappedLine();
524  }
525  // FIXME: Add error handling.
526}
527
528void UnwrappedLineParser::parseForOrWhileLoop() {
529  assert((FormatTok.Tok.is(tok::kw_for) || FormatTok.Tok.is(tok::kw_while)) &&
530         "'for' or 'while' expected");
531  nextToken();
532  if (FormatTok.Tok.is(tok::l_paren))
533    parseParens();
534  if (FormatTok.Tok.is(tok::l_brace)) {
535    parseBlock(/*MustBeDeclaration=*/ false);
536    addUnwrappedLine();
537  } else {
538    addUnwrappedLine();
539    ++Line->Level;
540    parseStructuralElement();
541    --Line->Level;
542  }
543}
544
545void UnwrappedLineParser::parseDoWhile() {
546  assert(FormatTok.Tok.is(tok::kw_do) && "'do' expected");
547  nextToken();
548  if (FormatTok.Tok.is(tok::l_brace)) {
549    parseBlock(/*MustBeDeclaration=*/ false);
550  } else {
551    addUnwrappedLine();
552    ++Line->Level;
553    parseStructuralElement();
554    --Line->Level;
555  }
556
557  // FIXME: Add error handling.
558  if (!FormatTok.Tok.is(tok::kw_while)) {
559    addUnwrappedLine();
560    return;
561  }
562
563  nextToken();
564  parseStructuralElement();
565}
566
567void UnwrappedLineParser::parseLabel() {
568  // FIXME: remove all asserts.
569  assert(FormatTok.Tok.is(tok::colon) && "':' expected");
570  nextToken();
571  unsigned OldLineLevel = Line->Level;
572  if (Line->Level > 0)
573    --Line->Level;
574  if (FormatTok.Tok.is(tok::l_brace)) {
575    parseBlock(/*MustBeDeclaration=*/ false);
576    if (FormatTok.Tok.is(tok::kw_break))
577      parseStructuralElement(); // "break;" after "}" goes on the same line.
578  }
579  addUnwrappedLine();
580  Line->Level = OldLineLevel;
581}
582
583void UnwrappedLineParser::parseCaseLabel() {
584  assert(FormatTok.Tok.is(tok::kw_case) && "'case' expected");
585  // FIXME: fix handling of complex expressions here.
586  do {
587    nextToken();
588  } while (!eof() && !FormatTok.Tok.is(tok::colon));
589  parseLabel();
590}
591
592void UnwrappedLineParser::parseSwitch() {
593  assert(FormatTok.Tok.is(tok::kw_switch) && "'switch' expected");
594  nextToken();
595  if (FormatTok.Tok.is(tok::l_paren))
596    parseParens();
597  if (FormatTok.Tok.is(tok::l_brace)) {
598    parseBlock(/*MustBeDeclaration=*/ false, Style.IndentCaseLabels ? 2 : 1);
599    addUnwrappedLine();
600  } else {
601    addUnwrappedLine();
602    Line->Level += (Style.IndentCaseLabels ? 2 : 1);
603    parseStructuralElement();
604    Line->Level -= (Style.IndentCaseLabels ? 2 : 1);
605  }
606}
607
608void UnwrappedLineParser::parseAccessSpecifier() {
609  nextToken();
610  // Otherwise, we don't know what it is, and we'd better keep the next token.
611  if (FormatTok.Tok.is(tok::colon))
612    nextToken();
613  addUnwrappedLine();
614}
615
616void UnwrappedLineParser::parseEnum() {
617  nextToken();
618  if (FormatTok.Tok.is(tok::identifier) ||
619      FormatTok.Tok.is(tok::kw___attribute) ||
620      FormatTok.Tok.is(tok::kw___declspec)) {
621    nextToken();
622    // We can have macros or attributes in between 'enum' and the enum name.
623    if (FormatTok.Tok.is(tok::l_paren)) {
624      parseParens();
625    }
626    if (FormatTok.Tok.is(tok::identifier))
627      nextToken();
628  }
629  if (FormatTok.Tok.is(tok::l_brace)) {
630    nextToken();
631    addUnwrappedLine();
632    ++Line->Level;
633    do {
634      switch (FormatTok.Tok.getKind()) {
635      case tok::l_paren:
636        parseParens();
637        break;
638      case tok::r_brace:
639        addUnwrappedLine();
640        nextToken();
641        --Line->Level;
642        return;
643      case tok::comma:
644        nextToken();
645        addUnwrappedLine();
646        break;
647      default:
648        nextToken();
649        break;
650      }
651    } while (!eof());
652  }
653  // We fall through to parsing a structural element afterwards, so that in
654  // enum A {} n, m;
655  // "} n, m;" will end up in one unwrapped line.
656}
657
658void UnwrappedLineParser::parseRecord() {
659  nextToken();
660  if (FormatTok.Tok.is(tok::identifier) ||
661      FormatTok.Tok.is(tok::kw___attribute) ||
662      FormatTok.Tok.is(tok::kw___declspec)) {
663    nextToken();
664    // We can have macros or attributes in between 'class' and the class name.
665    if (FormatTok.Tok.is(tok::l_paren)) {
666      parseParens();
667    }
668    // The actual identifier can be a nested name specifier, and in macros
669    // it is often token-pasted.
670    while (FormatTok.Tok.is(tok::identifier) ||
671           FormatTok.Tok.is(tok::coloncolon) ||
672           FormatTok.Tok.is(tok::hashhash))
673      nextToken();
674
675    // Note that parsing away template declarations here leads to incorrectly
676    // accepting function declarations as record declarations.
677    // In general, we cannot solve this problem. Consider:
678    // class A<int> B() {}
679    // which can be a function definition or a class definition when B() is a
680    // macro. If we find enough real-world cases where this is a problem, we
681    // can parse for the 'template' keyword in the beginning of the statement,
682    // and thus rule out the record production in case there is no template
683    // (this would still leave us with an ambiguity between template function
684    // and class declarations).
685    if (FormatTok.Tok.is(tok::colon) || FormatTok.Tok.is(tok::less)) {
686      while (FormatTok.Tok.isNot(tok::l_brace)) {
687        if (FormatTok.Tok.is(tok::semi))
688          return;
689        nextToken();
690      }
691    }
692  }
693  if (FormatTok.Tok.is(tok::l_brace))
694    parseBlock(/*MustBeDeclaration=*/ true);
695  // We fall through to parsing a structural element afterwards, so
696  // class A {} n, m;
697  // will end up in one unwrapped line.
698}
699
700void UnwrappedLineParser::parseObjCProtocolList() {
701  assert(FormatTok.Tok.is(tok::less) && "'<' expected.");
702  do
703    nextToken();
704  while (!eof() && FormatTok.Tok.isNot(tok::greater));
705  nextToken(); // Skip '>'.
706}
707
708void UnwrappedLineParser::parseObjCUntilAtEnd() {
709  do {
710    if (FormatTok.Tok.isObjCAtKeyword(tok::objc_end)) {
711      nextToken();
712      addUnwrappedLine();
713      break;
714    }
715    parseStructuralElement();
716  } while (!eof());
717}
718
719void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
720  nextToken();
721  nextToken();  // interface name
722
723  // @interface can be followed by either a base class, or a category.
724  if (FormatTok.Tok.is(tok::colon)) {
725    nextToken();
726    nextToken();  // base class name
727  } else if (FormatTok.Tok.is(tok::l_paren))
728    // Skip category, if present.
729    parseParens();
730
731  if (FormatTok.Tok.is(tok::less))
732    parseObjCProtocolList();
733
734  // If instance variables are present, keep the '{' on the first line too.
735  if (FormatTok.Tok.is(tok::l_brace))
736    parseBlock(/*MustBeDeclaration=*/ true);
737
738  // With instance variables, this puts '}' on its own line.  Without instance
739  // variables, this ends the @interface line.
740  addUnwrappedLine();
741
742  parseObjCUntilAtEnd();
743}
744
745void UnwrappedLineParser::parseObjCProtocol() {
746  nextToken();
747  nextToken();  // protocol name
748
749  if (FormatTok.Tok.is(tok::less))
750    parseObjCProtocolList();
751
752  // Check for protocol declaration.
753  if (FormatTok.Tok.is(tok::semi)) {
754    nextToken();
755    return addUnwrappedLine();
756  }
757
758  addUnwrappedLine();
759  parseObjCUntilAtEnd();
760}
761
762void UnwrappedLineParser::addUnwrappedLine() {
763  if (Line->Tokens.empty())
764    return;
765  DEBUG({
766    llvm::dbgs() << "Line(" << Line->Level << ")"
767                 << (Line->InPPDirective ? " MACRO" : "") << ": ";
768    for (std::list<FormatToken>::iterator I = Line->Tokens.begin(),
769                                          E = Line->Tokens.end();
770         I != E; ++I) {
771      llvm::dbgs() << I->Tok.getName() << " ";
772
773    }
774    llvm::dbgs() << "\n";
775  });
776  CurrentLines->push_back(*Line);
777  Line->Tokens.clear();
778  if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
779    for (std::vector<UnwrappedLine>::iterator I = PreprocessorDirectives
780             .begin(), E = PreprocessorDirectives.end();
781         I != E; ++I) {
782      CurrentLines->push_back(*I);
783    }
784    PreprocessorDirectives.clear();
785  }
786
787}
788
789bool UnwrappedLineParser::eof() const {
790  return FormatTok.Tok.is(tok::eof);
791}
792
793void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
794  bool JustComments = Line->Tokens.empty();
795  for (SmallVectorImpl<FormatToken>::const_iterator
796           I = CommentsBeforeNextToken.begin(),
797           E = CommentsBeforeNextToken.end();
798       I != E; ++I) {
799    if (I->NewlinesBefore && JustComments) {
800      addUnwrappedLine();
801    }
802    pushToken(*I);
803  }
804  if (NewlineBeforeNext && JustComments) {
805    addUnwrappedLine();
806  }
807  CommentsBeforeNextToken.clear();
808}
809
810void UnwrappedLineParser::nextToken() {
811  if (eof())
812    return;
813  flushComments(FormatTok.NewlinesBefore > 0);
814  pushToken(FormatTok);
815  readToken();
816}
817
818void UnwrappedLineParser::readToken() {
819  bool CommentsInCurrentLine = true;
820  do {
821    FormatTok = Tokens->getNextToken();
822    while (!Line->InPPDirective && FormatTok.Tok.is(tok::hash) &&
823           ((FormatTok.NewlinesBefore > 0 && FormatTok.HasUnescapedNewline) ||
824            FormatTok.IsFirst)) {
825      // If there is an unfinished unwrapped line, we flush the preprocessor
826      // directives only after that unwrapped line was finished later.
827      bool SwitchToPreprocessorLines = !Line->Tokens.empty() &&
828                                       CurrentLines == &Lines;
829      ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
830      parsePPDirective();
831    }
832    if (!FormatTok.Tok.is(tok::comment))
833      return;
834    if (FormatTok.NewlinesBefore > 0 || FormatTok.IsFirst) {
835      CommentsInCurrentLine = false;
836    }
837    if (CommentsInCurrentLine) {
838      pushToken(FormatTok);
839    } else {
840      CommentsBeforeNextToken.push_back(FormatTok);
841    }
842  } while (!eof());
843}
844
845void UnwrappedLineParser::pushToken(const FormatToken &Tok) {
846  Line->Tokens.push_back(Tok);
847  if (MustBreakBeforeNextToken) {
848    Line->Tokens.back().MustBreakBefore = true;
849    MustBreakBeforeNextToken = false;
850  }
851}
852
853} // end namespace format
854} // end namespace clang
855