UnwrappedLineParser.cpp revision 7ccbc2156bfe473f080b585130d70e51c4b62bb0 
1//===--- UnwrappedLineParser.cpp - Format C++ code ------------------------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9///
10/// \file
11/// \brief This file contains the implementation of the UnwrappedLineParser,
12/// which turns a stream of tokens into UnwrappedLines.
13///
14/// This is EXPERIMENTAL code under heavy development. It is not in a state yet,
15/// where it can be used to format real code.
16///
17//===----------------------------------------------------------------------===//
18
19#define DEBUG_TYPE "format-parser"
20
21#include "UnwrappedLineParser.h"
22#include "clang/Basic/Diagnostic.h"
23#include "llvm/Support/Debug.h"
24
25// Uncomment to get debug output from tests:
26// #define DEBUG_WITH_TYPE(T, X) do { X; } while(0)
27
28namespace clang {
29namespace format {
30
31class ScopedDeclarationState {
32public:
33  ScopedDeclarationState(UnwrappedLine &Line, std::vector<bool> &Stack,
34                         bool MustBeDeclaration)
35      : Line(Line), Stack(Stack) {
36    Line.MustBeDeclaration = MustBeDeclaration;
37    Stack.push_back(MustBeDeclaration);
38  }
39  ~ScopedDeclarationState() {
40    Stack.pop_back();
41    if (!Stack.empty())
42      Line.MustBeDeclaration = Stack.back();
43    else
44      Line.MustBeDeclaration = true;
45  }
46private:
47  UnwrappedLine &Line;
48  std::vector<bool> &Stack;
49};
50
51class ScopedMacroState : public FormatTokenSource {
52public:
53  ScopedMacroState(UnwrappedLine &Line, FormatTokenSource *&TokenSource,
54                   FormatToken &ResetToken)
55      : Line(Line), TokenSource(TokenSource), ResetToken(ResetToken),
56        PreviousLineLevel(Line.Level), PreviousTokenSource(TokenSource) {
57    TokenSource = this;
58    Line.Level = 0;
59    Line.InPPDirective = true;
60  }
61
62  ~ScopedMacroState() {
63    TokenSource = PreviousTokenSource;
64    ResetToken = Token;
65    Line.InPPDirective = false;
66    Line.Level = PreviousLineLevel;
67  }
68
69  virtual FormatToken getNextToken() {
70    // The \c UnwrappedLineParser guards against this by never calling
71    // \c getNextToken() after it has encountered the first eof token.
72    assert(!eof());
73    Token = PreviousTokenSource->getNextToken();
74    if (eof())
75      return createEOF();
76    return Token;
77  }
78
79private:
80  bool eof() {
81    return Token.NewlinesBefore > 0 && Token.HasUnescapedNewline;
82  }
83
84  FormatToken createEOF() {
85    FormatToken FormatTok;
86    FormatTok.Tok.startToken();
87    FormatTok.Tok.setKind(tok::eof);
88    return FormatTok;
89  }
90
91  UnwrappedLine &Line;
92  FormatTokenSource *&TokenSource;
93  FormatToken &ResetToken;
94  unsigned PreviousLineLevel;
95  FormatTokenSource *PreviousTokenSource;
96
97  FormatToken Token;
98};
99
100class ScopedLineState {
101public:
102  ScopedLineState(UnwrappedLineParser &Parser,
103                  bool SwitchToPreprocessorLines = false)
104      : Parser(Parser), SwitchToPreprocessorLines(SwitchToPreprocessorLines) {
105    if (SwitchToPreprocessorLines)
106      Parser.CurrentLines = &Parser.PreprocessorDirectives;
107    PreBlockLine = Parser.Line.take();
108    Parser.Line.reset(new UnwrappedLine());
109    Parser.Line->Level = PreBlockLine->Level;
110    Parser.Line->InPPDirective = PreBlockLine->InPPDirective;
111  }
112
113  ~ScopedLineState() {
114    if (!Parser.Line->Tokens.empty()) {
115      Parser.addUnwrappedLine();
116    }
117    assert(Parser.Line->Tokens.empty());
118    Parser.Line.reset(PreBlockLine);
119    Parser.MustBreakBeforeNextToken = true;
120    if (SwitchToPreprocessorLines)
121      Parser.CurrentLines = &Parser.Lines;
122  }
123
124private:
125  UnwrappedLineParser &Parser;
126  const bool SwitchToPreprocessorLines;
127
128  UnwrappedLine *PreBlockLine;
129};
130
131UnwrappedLineParser::UnwrappedLineParser(
132    clang::DiagnosticsEngine &Diag, const FormatStyle &Style,
133    FormatTokenSource &Tokens, UnwrappedLineConsumer &Callback)
134    : Line(new UnwrappedLine), MustBreakBeforeNextToken(false),
135      CurrentLines(&Lines), Diag(Diag), Style(Style), Tokens(&Tokens),
136      Callback(Callback) {}
137
138bool UnwrappedLineParser::parse() {
139  DEBUG(llvm::dbgs() << "----\n");
140  readToken();
141  bool Error = parseFile();
142  for (std::vector<UnwrappedLine>::iterator I = Lines.begin(),
143                                            E = Lines.end();
144       I != E; ++I) {
145    Callback.consumeUnwrappedLine(*I);
146  }
147  return Error;
148}
149
150bool UnwrappedLineParser::parseFile() {
151  ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
152                                          /*MustBeDeclaration=*/ true);
153  bool Error = parseLevel(/*HasOpeningBrace=*/false);
154  // Make sure to format the remaining tokens.
155  flushComments(true);
156  addUnwrappedLine();
157  return Error;
158}
159
160bool UnwrappedLineParser::parseLevel(bool HasOpeningBrace) {
161  bool Error = false;
162  do {
163    switch (FormatTok.Tok.getKind()) {
164    case tok::comment:
165      nextToken();
166      addUnwrappedLine();
167      break;
168    case tok::l_brace:
169      // FIXME: Add parameter whether this can happen - if this happens, we must
170      // be in a non-declaration context.
171      Error |= parseBlock(/*MustBeDeclaration=*/ false);
172      addUnwrappedLine();
173      break;
174    case tok::r_brace:
175      if (HasOpeningBrace) {
176        return false;
177      } else {
178        Diag.Report(FormatTok.Tok.getLocation(),
179                    Diag.getCustomDiagID(clang::DiagnosticsEngine::Error,
180                                         "unexpected '}'"));
181        Error = true;
182        nextToken();
183        addUnwrappedLine();
184      }
185      break;
186    default:
187      parseStructuralElement();
188      break;
189    }
190  } while (!eof());
191  return Error;
192}
193
194bool UnwrappedLineParser::parseBlock(bool MustBeDeclaration, unsigned AddLevels) {
195  assert(FormatTok.Tok.is(tok::l_brace) && "'{' expected");
196  nextToken();
197
198  addUnwrappedLine();
199
200  ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
201                                          MustBeDeclaration);
202  Line->Level += AddLevels;
203  parseLevel(/*HasOpeningBrace=*/true);
204
205  if (!FormatTok.Tok.is(tok::r_brace)) {
206    Line->Level -= AddLevels;
207    return true;
208  }
209
210  nextToken();  // Munch the closing brace.
211  Line->Level -= AddLevels;
212  return false;
213}
214
215void UnwrappedLineParser::parsePPDirective() {
216  assert(FormatTok.Tok.is(tok::hash) && "'#' expected");
217  ScopedMacroState MacroState(*Line, Tokens, FormatTok);
218  nextToken();
219
220  if (FormatTok.Tok.getIdentifierInfo() == NULL) {
221    addUnwrappedLine();
222    return;
223  }
224
225  switch (FormatTok.Tok.getIdentifierInfo()->getPPKeywordID()) {
226  case tok::pp_define:
227    parsePPDefine();
228    break;
229  default:
230    parsePPUnknown();
231    break;
232  }
233}
234
235void UnwrappedLineParser::parsePPDefine() {
236  nextToken();
237
238  if (FormatTok.Tok.getKind() != tok::identifier) {
239    parsePPUnknown();
240    return;
241  }
242  nextToken();
243  if (FormatTok.Tok.getKind() == tok::l_paren &&
244      FormatTok.WhiteSpaceLength == 0) {
245    parseParens();
246  }
247  addUnwrappedLine();
248  Line->Level = 1;
249
250  // Errors during a preprocessor directive can only affect the layout of the
251  // preprocessor directive, and thus we ignore them. An alternative approach
252  // would be to use the same approach we use on the file level (no
253  // re-indentation if there was a structural error) within the macro
254  // definition.
255  parseFile();
256}
257
258void UnwrappedLineParser::parsePPUnknown() {
259  do {
260    nextToken();
261  } while (!eof());
262  addUnwrappedLine();
263}
264
265void UnwrappedLineParser::parseStructuralElement() {
266  assert(!FormatTok.Tok.is(tok::l_brace));
267  int TokenNumber = 0;
268  switch (FormatTok.Tok.getKind()) {
269  case tok::at:
270    nextToken();
271    switch (FormatTok.Tok.getObjCKeywordID()) {
272    case tok::objc_public:
273    case tok::objc_protected:
274    case tok::objc_package:
275    case tok::objc_private:
276      return parseAccessSpecifier();
277    case tok::objc_interface:
278    case tok::objc_implementation:
279      return parseObjCInterfaceOrImplementation();
280    case tok::objc_protocol:
281      return parseObjCProtocol();
282    case tok::objc_end:
283      return; // Handled by the caller.
284    case tok::objc_optional:
285    case tok::objc_required:
286      nextToken();
287      addUnwrappedLine();
288      return;
289    default:
290      break;
291    }
292    break;
293  case tok::kw_namespace:
294    parseNamespace();
295    return;
296  case tok::kw_inline:
297    nextToken();
298    TokenNumber++;
299    if (FormatTok.Tok.is(tok::kw_namespace)) {
300      parseNamespace();
301      return;
302    }
303    break;
304  case tok::kw_public:
305  case tok::kw_protected:
306  case tok::kw_private:
307    parseAccessSpecifier();
308    return;
309  case tok::kw_if:
310    parseIfThenElse();
311    return;
312  case tok::kw_for:
313  case tok::kw_while:
314    parseForOrWhileLoop();
315    return;
316  case tok::kw_do:
317    parseDoWhile();
318    return;
319  case tok::kw_switch:
320    parseSwitch();
321    return;
322  case tok::kw_default:
323    nextToken();
324    parseLabel();
325    return;
326  case tok::kw_case:
327    parseCaseLabel();
328    return;
329  case tok::kw_return:
330    parseReturn();
331    return;
332  case tok::kw_extern:
333    nextToken();
334    if (FormatTok.Tok.is(tok::string_literal)) {
335      nextToken();
336      if (FormatTok.Tok.is(tok::l_brace)) {
337        parseBlock(/*MustBeDeclaration=*/ true, 0);
338        addUnwrappedLine();
339        return;
340      }
341    }
342    // In all other cases, parse the declaration.
343    break;
344  default:
345    break;
346  }
347  do {
348    ++TokenNumber;
349    switch (FormatTok.Tok.getKind()) {
350    case tok::kw_enum:
351      parseEnum();
352      break;
353    case tok::kw_struct:
354    case tok::kw_union:
355    case tok::kw_class:
356      parseRecord();
357      // A record declaration or definition is always the start of a structural
358      // element.
359      break;
360    case tok::semi:
361      nextToken();
362      addUnwrappedLine();
363      return;
364    case tok::r_brace:
365      addUnwrappedLine();
366      return;
367    case tok::l_paren:
368      parseParens();
369      break;
370    case tok::l_brace:
371      // A block outside of parentheses must be the last part of a
372      // structural element.
373      // FIXME: Figure out cases where this is not true, and add projections for
374      // them (the one we know is missing are lambdas).
375      parseBlock(/*MustBeDeclaration=*/ false);
376      addUnwrappedLine();
377      return;
378    case tok::identifier:
379      nextToken();
380      if (TokenNumber == 1 && FormatTok.Tok.is(tok::colon)) {
381        parseLabel();
382        return;
383      }
384      break;
385    case tok::equal:
386      nextToken();
387      if (FormatTok.Tok.is(tok::l_brace)) {
388        parseBracedList();
389      }
390      break;
391    default:
392      nextToken();
393      break;
394    }
395  } while (!eof());
396}
397
398void UnwrappedLineParser::parseBracedList() {
399  nextToken();
400
401  do {
402    switch (FormatTok.Tok.getKind()) {
403    case tok::l_brace:
404      parseBracedList();
405      break;
406    case tok::r_brace:
407      nextToken();
408      return;
409    default:
410      nextToken();
411      break;
412    }
413  } while (!eof());
414}
415
416void UnwrappedLineParser::parseReturn() {
417  nextToken();
418
419  do {
420    switch (FormatTok.Tok.getKind()) {
421    case tok::l_brace:
422      parseBracedList();
423      break;
424    case tok::l_paren:
425      parseParens();
426      break;
427    case tok::r_brace:
428      // Assume missing ';'.
429      addUnwrappedLine();
430      return;
431    case tok::semi:
432      nextToken();
433      addUnwrappedLine();
434      return;
435    default:
436      nextToken();
437      break;
438    }
439  } while (!eof());
440}
441
442void UnwrappedLineParser::parseParens() {
443  assert(FormatTok.Tok.is(tok::l_paren) && "'(' expected.");
444  nextToken();
445  do {
446    switch (FormatTok.Tok.getKind()) {
447    case tok::l_paren:
448      parseParens();
449      break;
450    case tok::r_paren:
451      nextToken();
452      return;
453    case tok::l_brace:
454      {
455        nextToken();
456        ScopedLineState LineState(*this);
457        ScopedDeclarationState DeclarationState(*Line, DeclarationScopeStack,
458                                                /*MustBeDeclaration=*/ false);
459        Line->Level += 1;
460        parseLevel(/*HasOpeningBrace=*/ true);
461        Line->Level -= 1;
462      }
463      break;
464    default:
465      nextToken();
466      break;
467    }
468  } while (!eof());
469}
470
471void UnwrappedLineParser::parseIfThenElse() {
472  assert(FormatTok.Tok.is(tok::kw_if) && "'if' expected");
473  nextToken();
474  if (FormatTok.Tok.is(tok::l_paren))
475    parseParens();
476  bool NeedsUnwrappedLine = false;
477  if (FormatTok.Tok.is(tok::l_brace)) {
478    parseBlock(/*MustBeDeclaration=*/ false);
479    NeedsUnwrappedLine = true;
480  } else {
481    addUnwrappedLine();
482    ++Line->Level;
483    parseStructuralElement();
484    --Line->Level;
485  }
486  if (FormatTok.Tok.is(tok::kw_else)) {
487    nextToken();
488    if (FormatTok.Tok.is(tok::l_brace)) {
489      parseBlock(/*MustBeDeclaration=*/ false);
490      addUnwrappedLine();
491    } else if (FormatTok.Tok.is(tok::kw_if)) {
492      parseIfThenElse();
493    } else {
494      addUnwrappedLine();
495      ++Line->Level;
496      parseStructuralElement();
497      --Line->Level;
498    }
499  } else if (NeedsUnwrappedLine) {
500    addUnwrappedLine();
501  }
502}
503
504void UnwrappedLineParser::parseNamespace() {
505  assert(FormatTok.Tok.is(tok::kw_namespace) && "'namespace' expected");
506  nextToken();
507  if (FormatTok.Tok.is(tok::identifier))
508    nextToken();
509  if (FormatTok.Tok.is(tok::l_brace)) {
510    parseBlock(/*MustBeDeclaration=*/ true, 0);
511    addUnwrappedLine();
512  }
513  // FIXME: Add error handling.
514}
515
516void UnwrappedLineParser::parseForOrWhileLoop() {
517  assert((FormatTok.Tok.is(tok::kw_for) || FormatTok.Tok.is(tok::kw_while)) &&
518         "'for' or 'while' expected");
519  nextToken();
520  if (FormatTok.Tok.is(tok::l_paren))
521    parseParens();
522  if (FormatTok.Tok.is(tok::l_brace)) {
523    parseBlock(/*MustBeDeclaration=*/ false);
524    addUnwrappedLine();
525  } else {
526    addUnwrappedLine();
527    ++Line->Level;
528    parseStructuralElement();
529    --Line->Level;
530  }
531}
532
533void UnwrappedLineParser::parseDoWhile() {
534  assert(FormatTok.Tok.is(tok::kw_do) && "'do' expected");
535  nextToken();
536  if (FormatTok.Tok.is(tok::l_brace)) {
537    parseBlock(/*MustBeDeclaration=*/ false);
538  } else {
539    addUnwrappedLine();
540    ++Line->Level;
541    parseStructuralElement();
542    --Line->Level;
543  }
544
545  // FIXME: Add error handling.
546  if (!FormatTok.Tok.is(tok::kw_while)) {
547    addUnwrappedLine();
548    return;
549  }
550
551  nextToken();
552  parseStructuralElement();
553}
554
555void UnwrappedLineParser::parseLabel() {
556  // FIXME: remove all asserts.
557  assert(FormatTok.Tok.is(tok::colon) && "':' expected");
558  nextToken();
559  unsigned OldLineLevel = Line->Level;
560  if (Line->Level > 0)
561    --Line->Level;
562  if (FormatTok.Tok.is(tok::l_brace)) {
563    parseBlock(/*MustBeDeclaration=*/ false);
564    if (FormatTok.Tok.is(tok::kw_break))
565      parseStructuralElement(); // "break;" after "}" goes on the same line.
566  }
567  addUnwrappedLine();
568  Line->Level = OldLineLevel;
569}
570
571void UnwrappedLineParser::parseCaseLabel() {
572  assert(FormatTok.Tok.is(tok::kw_case) && "'case' expected");
573  // FIXME: fix handling of complex expressions here.
574  do {
575    nextToken();
576  } while (!eof() && !FormatTok.Tok.is(tok::colon));
577  parseLabel();
578}
579
580void UnwrappedLineParser::parseSwitch() {
581  assert(FormatTok.Tok.is(tok::kw_switch) && "'switch' expected");
582  nextToken();
583  if (FormatTok.Tok.is(tok::l_paren))
584    parseParens();
585  if (FormatTok.Tok.is(tok::l_brace)) {
586    parseBlock(/*MustBeDeclaration=*/ false, Style.IndentCaseLabels ? 2 : 1);
587    addUnwrappedLine();
588  } else {
589    addUnwrappedLine();
590    Line->Level += (Style.IndentCaseLabels ? 2 : 1);
591    parseStructuralElement();
592    Line->Level -= (Style.IndentCaseLabels ? 2 : 1);
593  }
594}
595
596void UnwrappedLineParser::parseAccessSpecifier() {
597  nextToken();
598  // Otherwise, we don't know what it is, and we'd better keep the next token.
599  if (FormatTok.Tok.is(tok::colon))
600    nextToken();
601  addUnwrappedLine();
602}
603
604void UnwrappedLineParser::parseEnum() {
605  nextToken();
606  if (FormatTok.Tok.is(tok::identifier) ||
607      FormatTok.Tok.is(tok::kw___attribute) ||
608      FormatTok.Tok.is(tok::kw___declspec)) {
609    nextToken();
610    // We can have macros or attributes in between 'enum' and the enum name.
611    if (FormatTok.Tok.is(tok::l_paren)) {
612      parseParens();
613    }
614    if (FormatTok.Tok.is(tok::identifier))
615      nextToken();
616  }
617  if (FormatTok.Tok.is(tok::l_brace)) {
618    nextToken();
619    addUnwrappedLine();
620    ++Line->Level;
621    do {
622      switch (FormatTok.Tok.getKind()) {
623      case tok::l_paren:
624        parseParens();
625        break;
626      case tok::r_brace:
627        addUnwrappedLine();
628        nextToken();
629        --Line->Level;
630        return;
631      case tok::comma:
632        nextToken();
633        addUnwrappedLine();
634        break;
635      default:
636        nextToken();
637        break;
638      }
639    } while (!eof());
640  }
641  // We fall through to parsing a structural element afterwards, so that in
642  // enum A {} n, m;
643  // "} n, m;" will end up in one unwrapped line.
644}
645
646void UnwrappedLineParser::parseRecord() {
647  nextToken();
648  if (FormatTok.Tok.is(tok::identifier) ||
649      FormatTok.Tok.is(tok::kw___attribute) ||
650      FormatTok.Tok.is(tok::kw___declspec)) {
651    nextToken();
652    // We can have macros or attributes in between 'class' and the class name.
653    if (FormatTok.Tok.is(tok::l_paren)) {
654      parseParens();
655    }
656    // The actual identifier can be a nested name specifier.
657    while (FormatTok.Tok.is(tok::identifier) ||
658           FormatTok.Tok.is(tok::coloncolon))
659      nextToken();
660
661    // Note that parsing away template declarations here leads to incorrectly
662    // accepting function declarations as record declarations.
663    // In general, we cannot solve this problem. Consider:
664    // class A<int> B() {}
665    // which can be a function definition or a class definition when B() is a
666    // macro. If we find enough real-world cases where this is a problem, we
667    // can parse for the 'template' keyword in the beginning of the statement,
668    // and thus rule out the record production in case there is no template
669    // (this would still leave us with an ambiguity between template function
670    // and class declarations).
671    if (FormatTok.Tok.is(tok::colon) || FormatTok.Tok.is(tok::less)) {
672      while (FormatTok.Tok.isNot(tok::l_brace)) {
673        if (FormatTok.Tok.is(tok::semi))
674          return;
675        nextToken();
676      }
677    }
678  }
679  if (FormatTok.Tok.is(tok::l_brace))
680    parseBlock(/*MustBeDeclaration=*/ true);
681  // We fall through to parsing a structural element afterwards, so
682  // class A {} n, m;
683  // will end up in one unwrapped line.
684}
685
686void UnwrappedLineParser::parseObjCProtocolList() {
687  assert(FormatTok.Tok.is(tok::less) && "'<' expected.");
688  do
689    nextToken();
690  while (!eof() && FormatTok.Tok.isNot(tok::greater));
691  nextToken(); // Skip '>'.
692}
693
694void UnwrappedLineParser::parseObjCUntilAtEnd() {
695  do {
696    if (FormatTok.Tok.isObjCAtKeyword(tok::objc_end)) {
697      nextToken();
698      addUnwrappedLine();
699      break;
700    }
701    parseStructuralElement();
702  } while (!eof());
703}
704
705void UnwrappedLineParser::parseObjCInterfaceOrImplementation() {
706  nextToken();
707  nextToken();  // interface name
708
709  // @interface can be followed by either a base class, or a category.
710  if (FormatTok.Tok.is(tok::colon)) {
711    nextToken();
712    nextToken();  // base class name
713  } else if (FormatTok.Tok.is(tok::l_paren))
714    // Skip category, if present.
715    parseParens();
716
717  if (FormatTok.Tok.is(tok::less))
718    parseObjCProtocolList();
719
720  // If instance variables are present, keep the '{' on the first line too.
721  if (FormatTok.Tok.is(tok::l_brace))
722    parseBlock(/*MustBeDeclaration=*/ true);
723
724  // With instance variables, this puts '}' on its own line.  Without instance
725  // variables, this ends the @interface line.
726  addUnwrappedLine();
727
728  parseObjCUntilAtEnd();
729}
730
731void UnwrappedLineParser::parseObjCProtocol() {
732  nextToken();
733  nextToken();  // protocol name
734
735  if (FormatTok.Tok.is(tok::less))
736    parseObjCProtocolList();
737
738  // Check for protocol declaration.
739  if (FormatTok.Tok.is(tok::semi)) {
740    nextToken();
741    return addUnwrappedLine();
742  }
743
744  addUnwrappedLine();
745  parseObjCUntilAtEnd();
746}
747
748void UnwrappedLineParser::addUnwrappedLine() {
749  if (Line->Tokens.empty())
750    return;
751  DEBUG({
752    llvm::dbgs() << "Line(" << Line->Level << "): ";
753    for (std::list<FormatToken>::iterator I = Line->Tokens.begin(),
754                                          E = Line->Tokens.end();
755         I != E; ++I) {
756      llvm::dbgs() << I->Tok.getName() << " ";
757
758    }
759    llvm::dbgs() << "\n";
760  });
761  CurrentLines->push_back(*Line);
762  Line->Tokens.clear();
763  if (CurrentLines == &Lines && !PreprocessorDirectives.empty()) {
764    for (std::vector<UnwrappedLine>::iterator I = PreprocessorDirectives
765             .begin(), E = PreprocessorDirectives.end();
766         I != E; ++I) {
767      CurrentLines->push_back(*I);
768    }
769    PreprocessorDirectives.clear();
770  }
771
772}
773
774bool UnwrappedLineParser::eof() const {
775  return FormatTok.Tok.is(tok::eof);
776}
777
778void UnwrappedLineParser::flushComments(bool NewlineBeforeNext) {
779  bool JustComments = Line->Tokens.empty();
780  for (SmallVectorImpl<FormatToken>::const_iterator
781           I = CommentsBeforeNextToken.begin(),
782           E = CommentsBeforeNextToken.end();
783       I != E; ++I) {
784    if (I->HasUnescapedNewline && JustComments) {
785      addUnwrappedLine();
786    }
787    pushToken(*I);
788  }
789  if (NewlineBeforeNext && JustComments) {
790    addUnwrappedLine();
791  }
792  CommentsBeforeNextToken.clear();
793}
794
795void UnwrappedLineParser::nextToken() {
796  if (eof())
797    return;
798  flushComments(FormatTok.HasUnescapedNewline);
799  pushToken(FormatTok);
800  readToken();
801}
802
803void UnwrappedLineParser::readToken() {
804  bool CommentsInCurrentLine = true;
805  do {
806    FormatTok = Tokens->getNextToken();
807    while (!Line->InPPDirective && FormatTok.Tok.is(tok::hash) &&
808           ((FormatTok.NewlinesBefore > 0 && FormatTok.HasUnescapedNewline) ||
809            FormatTok.IsFirst)) {
810      // If there is an unfinished unwrapped line, we flush the preprocessor
811      // directives only after that unwrapped line was finished later.
812      bool SwitchToPreprocessorLines = !Line->Tokens.empty() &&
813                                       CurrentLines == &Lines;
814      ScopedLineState BlockState(*this, SwitchToPreprocessorLines);
815      parsePPDirective();
816    }
817    if (!FormatTok.Tok.is(tok::comment))
818      return;
819    if (FormatTok.HasUnescapedNewline || FormatTok.IsFirst) {
820      CommentsInCurrentLine = false;
821    }
822    if (CommentsInCurrentLine) {
823      pushToken(FormatTok);
824    } else {
825      CommentsBeforeNextToken.push_back(FormatTok);
826    }
827  } while (!eof());
828}
829
830void UnwrappedLineParser::pushToken(const FormatToken &Tok) {
831  Line->Tokens.push_back(Tok);
832  if (MustBreakBeforeNextToken) {
833    Line->Tokens.back().MustBreakBefore = true;
834    MustBreakBeforeNextToken = false;
835  }
836}
837
838} // end namespace format
839} // end namespace clang
840