1//== HTMLRewrite.cpp - Translate source code into prettified HTML --*- C++ -*-//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10//  This file defines the HTMLRewriter class, which is used to translate the
11//  text of a source file into prettified HTML.
12//
13//===----------------------------------------------------------------------===//
14
15#include "clang/Rewrite/Core/HTMLRewrite.h"
16#include "clang/Basic/SourceManager.h"
17#include "clang/Lex/Preprocessor.h"
18#include "clang/Lex/TokenConcatenation.h"
19#include "clang/Rewrite/Core/Rewriter.h"
20#include "llvm/ADT/SmallString.h"
21#include "llvm/Support/ErrorHandling.h"
22#include "llvm/Support/MemoryBuffer.h"
23#include "llvm/Support/raw_ostream.h"
24#include <memory>
25using namespace clang;
26
27
28/// HighlightRange - Highlight a range in the source code with the specified
29/// start/end tags.  B/E must be in the same file.  This ensures that
30/// start/end tags are placed at the start/end of each line if the range is
31/// multiline.
32void html::HighlightRange(Rewriter &R, SourceLocation B, SourceLocation E,
33                          const char *StartTag, const char *EndTag) {
34  SourceManager &SM = R.getSourceMgr();
35  B = SM.getExpansionLoc(B);
36  E = SM.getExpansionLoc(E);
37  FileID FID = SM.getFileID(B);
38  assert(SM.getFileID(E) == FID && "B/E not in the same file!");
39
40  unsigned BOffset = SM.getFileOffset(B);
41  unsigned EOffset = SM.getFileOffset(E);
42
43  // Include the whole end token in the range.
44  EOffset += Lexer::MeasureTokenLength(E, R.getSourceMgr(), R.getLangOpts());
45
46  bool Invalid = false;
47  const char *BufferStart = SM.getBufferData(FID, &Invalid).data();
48  if (Invalid)
49    return;
50
51  HighlightRange(R.getEditBuffer(FID), BOffset, EOffset,
52                 BufferStart, StartTag, EndTag);
53}
54
55/// HighlightRange - This is the same as the above method, but takes
56/// decomposed file locations.
57void html::HighlightRange(RewriteBuffer &RB, unsigned B, unsigned E,
58                          const char *BufferStart,
59                          const char *StartTag, const char *EndTag) {
60  // Insert the tag at the absolute start/end of the range.
61  RB.InsertTextAfter(B, StartTag);
62  RB.InsertTextBefore(E, EndTag);
63
64  // Scan the range to see if there is a \r or \n.  If so, and if the line is
65  // not blank, insert tags on that line as well.
66  bool HadOpenTag = true;
67
68  unsigned LastNonWhiteSpace = B;
69  for (unsigned i = B; i != E; ++i) {
70    switch (BufferStart[i]) {
71    case '\r':
72    case '\n':
73      // Okay, we found a newline in the range.  If we have an open tag, we need
74      // to insert a close tag at the first non-whitespace before the newline.
75      if (HadOpenTag)
76        RB.InsertTextBefore(LastNonWhiteSpace+1, EndTag);
77
78      // Instead of inserting an open tag immediately after the newline, we
79      // wait until we see a non-whitespace character.  This prevents us from
80      // inserting tags around blank lines, and also allows the open tag to
81      // be put *after* whitespace on a non-blank line.
82      HadOpenTag = false;
83      break;
84    case '\0':
85    case ' ':
86    case '\t':
87    case '\f':
88    case '\v':
89      // Ignore whitespace.
90      break;
91
92    default:
93      // If there is no tag open, do it now.
94      if (!HadOpenTag) {
95        RB.InsertTextAfter(i, StartTag);
96        HadOpenTag = true;
97      }
98
99      // Remember this character.
100      LastNonWhiteSpace = i;
101      break;
102    }
103  }
104}
105
106void html::EscapeText(Rewriter &R, FileID FID,
107                      bool EscapeSpaces, bool ReplaceTabs) {
108
109  const llvm::MemoryBuffer *Buf = R.getSourceMgr().getBuffer(FID);
110  const char* C = Buf->getBufferStart();
111  const char* FileEnd = Buf->getBufferEnd();
112
113  assert (C <= FileEnd);
114
115  RewriteBuffer &RB = R.getEditBuffer(FID);
116
117  unsigned ColNo = 0;
118  for (unsigned FilePos = 0; C != FileEnd ; ++C, ++FilePos) {
119    switch (*C) {
120    default: ++ColNo; break;
121    case '\n':
122    case '\r':
123      ColNo = 0;
124      break;
125
126    case ' ':
127      if (EscapeSpaces)
128        RB.ReplaceText(FilePos, 1, "&nbsp;");
129      ++ColNo;
130      break;
131    case '\f':
132      RB.ReplaceText(FilePos, 1, "<hr>");
133      ColNo = 0;
134      break;
135
136    case '\t': {
137      if (!ReplaceTabs)
138        break;
139      unsigned NumSpaces = 8-(ColNo&7);
140      if (EscapeSpaces)
141        RB.ReplaceText(FilePos, 1,
142                       StringRef("&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;"
143                                       "&nbsp;&nbsp;&nbsp;", 6*NumSpaces));
144      else
145        RB.ReplaceText(FilePos, 1, StringRef("        ", NumSpaces));
146      ColNo += NumSpaces;
147      break;
148    }
149    case '<':
150      RB.ReplaceText(FilePos, 1, "&lt;");
151      ++ColNo;
152      break;
153
154    case '>':
155      RB.ReplaceText(FilePos, 1, "&gt;");
156      ++ColNo;
157      break;
158
159    case '&':
160      RB.ReplaceText(FilePos, 1, "&amp;");
161      ++ColNo;
162      break;
163    }
164  }
165}
166
167std::string html::EscapeText(StringRef s, bool EscapeSpaces, bool ReplaceTabs) {
168
169  unsigned len = s.size();
170  std::string Str;
171  llvm::raw_string_ostream os(Str);
172
173  for (unsigned i = 0 ; i < len; ++i) {
174
175    char c = s[i];
176    switch (c) {
177    default:
178      os << c; break;
179
180    case ' ':
181      if (EscapeSpaces) os << "&nbsp;";
182      else os << ' ';
183      break;
184
185    case '\t':
186      if (ReplaceTabs) {
187        if (EscapeSpaces)
188          for (unsigned i = 0; i < 4; ++i)
189            os << "&nbsp;";
190        else
191          for (unsigned i = 0; i < 4; ++i)
192            os << " ";
193      }
194      else
195        os << c;
196
197      break;
198
199    case '<': os << "&lt;"; break;
200    case '>': os << "&gt;"; break;
201    case '&': os << "&amp;"; break;
202    }
203  }
204
205  return os.str();
206}
207
208static void AddLineNumber(RewriteBuffer &RB, unsigned LineNo,
209                          unsigned B, unsigned E) {
210  SmallString<256> Str;
211  llvm::raw_svector_ostream OS(Str);
212
213  OS << "<tr><td class=\"num\" id=\"LN"
214     << LineNo << "\">"
215     << LineNo << "</td><td class=\"line\">";
216
217  if (B == E) { // Handle empty lines.
218    OS << " </td></tr>";
219    RB.InsertTextBefore(B, OS.str());
220  } else {
221    RB.InsertTextBefore(B, OS.str());
222    RB.InsertTextBefore(E, "</td></tr>");
223  }
224}
225
226void html::AddLineNumbers(Rewriter& R, FileID FID) {
227
228  const llvm::MemoryBuffer *Buf = R.getSourceMgr().getBuffer(FID);
229  const char* FileBeg = Buf->getBufferStart();
230  const char* FileEnd = Buf->getBufferEnd();
231  const char* C = FileBeg;
232  RewriteBuffer &RB = R.getEditBuffer(FID);
233
234  assert (C <= FileEnd);
235
236  unsigned LineNo = 0;
237  unsigned FilePos = 0;
238
239  while (C != FileEnd) {
240
241    ++LineNo;
242    unsigned LineStartPos = FilePos;
243    unsigned LineEndPos = FileEnd - FileBeg;
244
245    assert (FilePos <= LineEndPos);
246    assert (C < FileEnd);
247
248    // Scan until the newline (or end-of-file).
249
250    while (C != FileEnd) {
251      char c = *C;
252      ++C;
253
254      if (c == '\n') {
255        LineEndPos = FilePos++;
256        break;
257      }
258
259      ++FilePos;
260    }
261
262    AddLineNumber(RB, LineNo, LineStartPos, LineEndPos);
263  }
264
265  // Add one big table tag that surrounds all of the code.
266  RB.InsertTextBefore(0, "<table class=\"code\">\n");
267  RB.InsertTextAfter(FileEnd - FileBeg, "</table>");
268}
269
270void html::AddHeaderFooterInternalBuiltinCSS(Rewriter& R, FileID FID,
271                                             const char *title) {
272
273  const llvm::MemoryBuffer *Buf = R.getSourceMgr().getBuffer(FID);
274  const char* FileStart = Buf->getBufferStart();
275  const char* FileEnd = Buf->getBufferEnd();
276
277  SourceLocation StartLoc = R.getSourceMgr().getLocForStartOfFile(FID);
278  SourceLocation EndLoc = StartLoc.getLocWithOffset(FileEnd-FileStart);
279
280  std::string s;
281  llvm::raw_string_ostream os(s);
282  os << "<!doctype html>\n" // Use HTML 5 doctype
283        "<html>\n<head>\n";
284
285  if (title)
286    os << "<title>" << html::EscapeText(title) << "</title>\n";
287
288  os << "<style type=\"text/css\">\n"
289      " body { color:#000000; background-color:#ffffff }\n"
290      " body { font-family:Helvetica, sans-serif; font-size:10pt }\n"
291      " h1 { font-size:14pt }\n"
292      " .code { border-collapse:collapse; width:100%; }\n"
293      " .code { font-family: \"Monospace\", monospace; font-size:10pt }\n"
294      " .code { line-height: 1.2em }\n"
295      " .comment { color: green; font-style: oblique }\n"
296      " .keyword { color: blue }\n"
297      " .string_literal { color: red }\n"
298      " .directive { color: darkmagenta }\n"
299      // Macro expansions.
300      " .expansion { display: none; }\n"
301      " .macro:hover .expansion { display: block; border: 2px solid #FF0000; "
302          "padding: 2px; background-color:#FFF0F0; font-weight: normal; "
303          "  -webkit-border-radius:5px;  -webkit-box-shadow:1px 1px 7px #000; "
304          "position: absolute; top: -1em; left:10em; z-index: 1 } \n"
305      " .macro { color: darkmagenta; background-color:LemonChiffon;"
306             // Macros are position: relative to provide base for expansions.
307             " position: relative }\n"
308      " .num { width:2.5em; padding-right:2ex; background-color:#eeeeee }\n"
309      " .num { text-align:right; font-size:8pt }\n"
310      " .num { color:#444444 }\n"
311      " .line { padding-left: 1ex; border-left: 3px solid #ccc }\n"
312      " .line { white-space: pre }\n"
313      " .msg { -webkit-box-shadow:1px 1px 7px #000 }\n"
314      " .msg { -webkit-border-radius:5px }\n"
315      " .msg { font-family:Helvetica, sans-serif; font-size:8pt }\n"
316      " .msg { float:left }\n"
317      " .msg { padding:0.25em 1ex 0.25em 1ex }\n"
318      " .msg { margin-top:10px; margin-bottom:10px }\n"
319      " .msg { font-weight:bold }\n"
320      " .msg { max-width:60em; word-wrap: break-word; white-space: pre-wrap }\n"
321      " .msgT { padding:0x; spacing:0x }\n"
322      " .msgEvent { background-color:#fff8b4; color:#000000 }\n"
323      " .msgControl { background-color:#bbbbbb; color:#000000 }\n"
324      " .mrange { background-color:#dfddf3 }\n"
325      " .mrange { border-bottom:1px solid #6F9DBE }\n"
326      " .PathIndex { font-weight: bold; padding:0px 5px; "
327        "margin-right:5px; }\n"
328      " .PathIndex { -webkit-border-radius:8px }\n"
329      " .PathIndexEvent { background-color:#bfba87 }\n"
330      " .PathIndexControl { background-color:#8c8c8c }\n"
331      " .PathNav a { text-decoration:none; font-size: larger }\n"
332      " .CodeInsertionHint { font-weight: bold; background-color: #10dd10 }\n"
333      " .CodeRemovalHint { background-color:#de1010 }\n"
334      " .CodeRemovalHint { border-bottom:1px solid #6F9DBE }\n"
335      " table.simpletable {\n"
336      "   padding: 5px;\n"
337      "   font-size:12pt;\n"
338      "   margin:20px;\n"
339      "   border-collapse: collapse; border-spacing: 0px;\n"
340      " }\n"
341      " td.rowname {\n"
342      "   text-align:right; font-weight:bold; color:#444444;\n"
343      "   padding-right:2ex; }\n"
344      "</style>\n</head>\n<body>";
345
346  // Generate header
347  R.InsertTextBefore(StartLoc, os.str());
348  // Generate footer
349
350  R.InsertTextAfter(EndLoc, "</body></html>\n");
351}
352
353/// SyntaxHighlight - Relex the specified FileID and annotate the HTML with
354/// information about keywords, macro expansions etc.  This uses the macro
355/// table state from the end of the file, so it won't be perfectly perfect,
356/// but it will be reasonably close.
357void html::SyntaxHighlight(Rewriter &R, FileID FID, const Preprocessor &PP) {
358  RewriteBuffer &RB = R.getEditBuffer(FID);
359
360  const SourceManager &SM = PP.getSourceManager();
361  const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
362  Lexer L(FID, FromFile, SM, PP.getLangOpts());
363  const char *BufferStart = L.getBuffer().data();
364
365  // Inform the preprocessor that we want to retain comments as tokens, so we
366  // can highlight them.
367  L.SetCommentRetentionState(true);
368
369  // Lex all the tokens in raw mode, to avoid entering #includes or expanding
370  // macros.
371  Token Tok;
372  L.LexFromRawLexer(Tok);
373
374  while (Tok.isNot(tok::eof)) {
375    // Since we are lexing unexpanded tokens, all tokens are from the main
376    // FileID.
377    unsigned TokOffs = SM.getFileOffset(Tok.getLocation());
378    unsigned TokLen = Tok.getLength();
379    switch (Tok.getKind()) {
380    default: break;
381    case tok::identifier:
382      llvm_unreachable("tok::identifier in raw lexing mode!");
383    case tok::raw_identifier: {
384      // Fill in Result.IdentifierInfo and update the token kind,
385      // looking up the identifier in the identifier table.
386      PP.LookUpIdentifierInfo(Tok);
387
388      // If this is a pp-identifier, for a keyword, highlight it as such.
389      if (Tok.isNot(tok::identifier))
390        HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart,
391                       "<span class='keyword'>", "</span>");
392      break;
393    }
394    case tok::comment:
395      HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart,
396                     "<span class='comment'>", "</span>");
397      break;
398    case tok::utf8_string_literal:
399      // Chop off the u part of u8 prefix
400      ++TokOffs;
401      --TokLen;
402      // FALL THROUGH to chop the 8
403    case tok::wide_string_literal:
404    case tok::utf16_string_literal:
405    case tok::utf32_string_literal:
406      // Chop off the L, u, U or 8 prefix
407      ++TokOffs;
408      --TokLen;
409      // FALL THROUGH.
410    case tok::string_literal:
411      // FIXME: Exclude the optional ud-suffix from the highlighted range.
412      HighlightRange(RB, TokOffs, TokOffs+TokLen, BufferStart,
413                     "<span class='string_literal'>", "</span>");
414      break;
415    case tok::hash: {
416      // If this is a preprocessor directive, all tokens to end of line are too.
417      if (!Tok.isAtStartOfLine())
418        break;
419
420      // Eat all of the tokens until we get to the next one at the start of
421      // line.
422      unsigned TokEnd = TokOffs+TokLen;
423      L.LexFromRawLexer(Tok);
424      while (!Tok.isAtStartOfLine() && Tok.isNot(tok::eof)) {
425        TokEnd = SM.getFileOffset(Tok.getLocation())+Tok.getLength();
426        L.LexFromRawLexer(Tok);
427      }
428
429      // Find end of line.  This is a hack.
430      HighlightRange(RB, TokOffs, TokEnd, BufferStart,
431                     "<span class='directive'>", "</span>");
432
433      // Don't skip the next token.
434      continue;
435    }
436    }
437
438    L.LexFromRawLexer(Tok);
439  }
440}
441
442/// HighlightMacros - This uses the macro table state from the end of the
443/// file, to re-expand macros and insert (into the HTML) information about the
444/// macro expansions.  This won't be perfectly perfect, but it will be
445/// reasonably close.
446void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) {
447  // Re-lex the raw token stream into a token buffer.
448  const SourceManager &SM = PP.getSourceManager();
449  std::vector<Token> TokenStream;
450
451  const llvm::MemoryBuffer *FromFile = SM.getBuffer(FID);
452  Lexer L(FID, FromFile, SM, PP.getLangOpts());
453
454  // Lex all the tokens in raw mode, to avoid entering #includes or expanding
455  // macros.
456  while (1) {
457    Token Tok;
458    L.LexFromRawLexer(Tok);
459
460    // If this is a # at the start of a line, discard it from the token stream.
461    // We don't want the re-preprocess step to see #defines, #includes or other
462    // preprocessor directives.
463    if (Tok.is(tok::hash) && Tok.isAtStartOfLine())
464      continue;
465
466    // If this is a ## token, change its kind to unknown so that repreprocessing
467    // it will not produce an error.
468    if (Tok.is(tok::hashhash))
469      Tok.setKind(tok::unknown);
470
471    // If this raw token is an identifier, the raw lexer won't have looked up
472    // the corresponding identifier info for it.  Do this now so that it will be
473    // macro expanded when we re-preprocess it.
474    if (Tok.is(tok::raw_identifier))
475      PP.LookUpIdentifierInfo(Tok);
476
477    TokenStream.push_back(Tok);
478
479    if (Tok.is(tok::eof)) break;
480  }
481
482  // Temporarily change the diagnostics object so that we ignore any generated
483  // diagnostics from this pass.
484  DiagnosticsEngine TmpDiags(PP.getDiagnostics().getDiagnosticIDs(),
485                             &PP.getDiagnostics().getDiagnosticOptions(),
486                      new IgnoringDiagConsumer);
487
488  // FIXME: This is a huge hack; we reuse the input preprocessor because we want
489  // its state, but we aren't actually changing it (we hope). This should really
490  // construct a copy of the preprocessor.
491  Preprocessor &TmpPP = const_cast<Preprocessor&>(PP);
492  DiagnosticsEngine *OldDiags = &TmpPP.getDiagnostics();
493  TmpPP.setDiagnostics(TmpDiags);
494
495  // Inform the preprocessor that we don't want comments.
496  TmpPP.SetCommentRetentionState(false, false);
497
498  // We don't want pragmas either. Although we filtered out #pragma, removing
499  // _Pragma and __pragma is much harder.
500  bool PragmasPreviouslyEnabled = TmpPP.getPragmasEnabled();
501  TmpPP.setPragmasEnabled(false);
502
503  // Enter the tokens we just lexed.  This will cause them to be macro expanded
504  // but won't enter sub-files (because we removed #'s).
505  TmpPP.EnterTokenStream(&TokenStream[0], TokenStream.size(), false, false);
506
507  TokenConcatenation ConcatInfo(TmpPP);
508
509  // Lex all the tokens.
510  Token Tok;
511  TmpPP.Lex(Tok);
512  while (Tok.isNot(tok::eof)) {
513    // Ignore non-macro tokens.
514    if (!Tok.getLocation().isMacroID()) {
515      TmpPP.Lex(Tok);
516      continue;
517    }
518
519    // Okay, we have the first token of a macro expansion: highlight the
520    // expansion by inserting a start tag before the macro expansion and
521    // end tag after it.
522    std::pair<SourceLocation, SourceLocation> LLoc =
523      SM.getExpansionRange(Tok.getLocation());
524
525    // Ignore tokens whose instantiation location was not the main file.
526    if (SM.getFileID(LLoc.first) != FID) {
527      TmpPP.Lex(Tok);
528      continue;
529    }
530
531    assert(SM.getFileID(LLoc.second) == FID &&
532           "Start and end of expansion must be in the same ultimate file!");
533
534    std::string Expansion = EscapeText(TmpPP.getSpelling(Tok));
535    unsigned LineLen = Expansion.size();
536
537    Token PrevPrevTok;
538    Token PrevTok = Tok;
539    // Okay, eat this token, getting the next one.
540    TmpPP.Lex(Tok);
541
542    // Skip all the rest of the tokens that are part of this macro
543    // instantiation.  It would be really nice to pop up a window with all the
544    // spelling of the tokens or something.
545    while (!Tok.is(tok::eof) &&
546           SM.getExpansionLoc(Tok.getLocation()) == LLoc.first) {
547      // Insert a newline if the macro expansion is getting large.
548      if (LineLen > 60) {
549        Expansion += "<br>";
550        LineLen = 0;
551      }
552
553      LineLen -= Expansion.size();
554
555      // If the tokens were already space separated, or if they must be to avoid
556      // them being implicitly pasted, add a space between them.
557      if (Tok.hasLeadingSpace() ||
558          ConcatInfo.AvoidConcat(PrevPrevTok, PrevTok, Tok))
559        Expansion += ' ';
560
561      // Escape any special characters in the token text.
562      Expansion += EscapeText(TmpPP.getSpelling(Tok));
563      LineLen += Expansion.size();
564
565      PrevPrevTok = PrevTok;
566      PrevTok = Tok;
567      TmpPP.Lex(Tok);
568    }
569
570
571    // Insert the expansion as the end tag, so that multi-line macros all get
572    // highlighted.
573    Expansion = "<span class='expansion'>" + Expansion + "</span></span>";
574
575    HighlightRange(R, LLoc.first, LLoc.second,
576                   "<span class='macro'>", Expansion.c_str());
577  }
578
579  // Restore the preprocessor's old state.
580  TmpPP.setDiagnostics(*OldDiags);
581  TmpPP.setPragmasEnabled(PragmasPreviouslyEnabled);
582}
583