LiteralSupport.cpp revision 4e93b34fdb798abfa0534062a139f2c37cbf876e 
1//===--- LiteralSupport.cpp - Code to parse and process literals ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the NumericLiteralParser, CharLiteralParser, and
11// StringLiteralParser interfaces.
12//
13//===----------------------------------------------------------------------===//
14
15#include "clang/Lex/LiteralSupport.h"
16#include "clang/Lex/Preprocessor.h"
17#include "clang/Lex/LexDiagnostic.h"
18#include "clang/Basic/TargetInfo.h"
19#include "llvm/ADT/StringExtras.h"
20using namespace clang;
21
22/// HexDigitValue - Return the value of the specified hex digit, or -1 if it's
23/// not valid.
24static int HexDigitValue(char C) {
25  if (C >= '0' && C <= '9') return C-'0';
26  if (C >= 'a' && C <= 'f') return C-'a'+10;
27  if (C >= 'A' && C <= 'F') return C-'A'+10;
28  return -1;
29}
30
31/// ProcessCharEscape - Parse a standard C escape sequence, which can occur in
32/// either a character or a string literal.
33static unsigned ProcessCharEscape(const char *&ThisTokBuf,
34                                  const char *ThisTokEnd, bool &HadError,
35                                  SourceLocation Loc, bool IsWide,
36                                  Preprocessor &PP) {
37  // Skip the '\' char.
38  ++ThisTokBuf;
39
40  // We know that this character can't be off the end of the buffer, because
41  // that would have been \", which would not have been the end of string.
42  unsigned ResultChar = *ThisTokBuf++;
43  switch (ResultChar) {
44  // These map to themselves.
45  case '\\': case '\'': case '"': case '?': break;
46
47    // These have fixed mappings.
48  case 'a':
49    // TODO: K&R: the meaning of '\\a' is different in traditional C
50    ResultChar = 7;
51    break;
52  case 'b':
53    ResultChar = 8;
54    break;
55  case 'e':
56    PP.Diag(Loc, diag::ext_nonstandard_escape) << "e";
57    ResultChar = 27;
58    break;
59  case 'f':
60    ResultChar = 12;
61    break;
62  case 'n':
63    ResultChar = 10;
64    break;
65  case 'r':
66    ResultChar = 13;
67    break;
68  case 't':
69    ResultChar = 9;
70    break;
71  case 'v':
72    ResultChar = 11;
73    break;
74  case 'x': { // Hex escape.
75    ResultChar = 0;
76    if (ThisTokBuf == ThisTokEnd || !isxdigit(*ThisTokBuf)) {
77      PP.Diag(Loc, diag::err_hex_escape_no_digits);
78      HadError = 1;
79      break;
80    }
81
82    // Hex escapes are a maximal series of hex digits.
83    bool Overflow = false;
84    for (; ThisTokBuf != ThisTokEnd; ++ThisTokBuf) {
85      int CharVal = HexDigitValue(ThisTokBuf[0]);
86      if (CharVal == -1) break;
87      // About to shift out a digit?
88      Overflow |= (ResultChar & 0xF0000000) ? true : false;
89      ResultChar <<= 4;
90      ResultChar |= CharVal;
91    }
92
93    // See if any bits will be truncated when evaluated as a character.
94    unsigned CharWidth = PP.getTargetInfo().getCharWidth(IsWide);
95
96    if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
97      Overflow = true;
98      ResultChar &= ~0U >> (32-CharWidth);
99    }
100
101    // Check for overflow.
102    if (Overflow)   // Too many digits to fit in
103      PP.Diag(Loc, diag::warn_hex_escape_too_large);
104    break;
105  }
106  case '0': case '1': case '2': case '3':
107  case '4': case '5': case '6': case '7': {
108    // Octal escapes.
109    --ThisTokBuf;
110    ResultChar = 0;
111
112    // Octal escapes are a series of octal digits with maximum length 3.
113    // "\0123" is a two digit sequence equal to "\012" "3".
114    unsigned NumDigits = 0;
115    do {
116      ResultChar <<= 3;
117      ResultChar |= *ThisTokBuf++ - '0';
118      ++NumDigits;
119    } while (ThisTokBuf != ThisTokEnd && NumDigits < 3 &&
120             ThisTokBuf[0] >= '0' && ThisTokBuf[0] <= '7');
121
122    // Check for overflow.  Reject '\777', but not L'\777'.
123    unsigned CharWidth = PP.getTargetInfo().getCharWidth(IsWide);
124
125    if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
126      PP.Diag(Loc, diag::warn_octal_escape_too_large);
127      ResultChar &= ~0U >> (32-CharWidth);
128    }
129    break;
130  }
131
132    // Otherwise, these are not valid escapes.
133  case '(': case '{': case '[': case '%':
134    // GCC accepts these as extensions.  We warn about them as such though.
135    if (!PP.getLangOptions().NoExtensions) {
136      PP.Diag(Loc, diag::ext_nonstandard_escape)
137        << std::string()+(char)ResultChar;
138      break;
139    }
140    // FALL THROUGH.
141  default:
142    if (isgraph(ThisTokBuf[0]))
143      PP.Diag(Loc, diag::ext_unknown_escape) << std::string()+(char)ResultChar;
144    else
145      PP.Diag(Loc, diag::ext_unknown_escape) << "x"+llvm::utohexstr(ResultChar);
146    break;
147  }
148
149  return ResultChar;
150}
151
152/// ProcessUCNEscape - Read the Universal Character Name, check constraints and
153/// convert the UTF32 to UTF8. This is a subroutine of StringLiteralParser.
154/// When we decide to implement UCN's for character constants and identifiers,
155/// we will likely rework our support for UCN's.
156static void ProcessUCNEscape(const char *&ThisTokBuf, const char *ThisTokEnd,
157                             char *&ResultBuf, bool &HadError,
158                             SourceLocation Loc, bool IsWide, Preprocessor &PP)
159{
160  // FIXME: Add a warning - UCN's are only valid in C++ & C99.
161  // FIXME: Handle wide strings.
162
163  // Save the beginning of the string (for error diagnostics).
164  const char *ThisTokBegin = ThisTokBuf;
165
166  // Skip the '\u' char's.
167  ThisTokBuf += 2;
168
169  if (ThisTokBuf == ThisTokEnd || !isxdigit(*ThisTokBuf)) {
170    PP.Diag(Loc, diag::err_ucn_escape_no_digits);
171    HadError = 1;
172    return;
173  }
174  typedef uint32_t UTF32;
175
176  UTF32 UcnVal = 0;
177  unsigned short UcnLen = (ThisTokBuf[-1] == 'u' ? 4 : 8);
178  for (; ThisTokBuf != ThisTokEnd && UcnLen; ++ThisTokBuf, UcnLen--) {
179    int CharVal = HexDigitValue(ThisTokBuf[0]);
180    if (CharVal == -1) break;
181    UcnVal <<= 4;
182    UcnVal |= CharVal;
183  }
184  // If we didn't consume the proper number of digits, there is a problem.
185  if (UcnLen) {
186    PP.Diag(PP.AdvanceToTokenCharacter(Loc, ThisTokBuf-ThisTokBegin),
187            diag::err_ucn_escape_incomplete);
188    HadError = 1;
189    return;
190  }
191  // Check UCN constraints (C99 6.4.3p2).
192  if ((UcnVal < 0xa0 &&
193      (UcnVal != 0x24 && UcnVal != 0x40 && UcnVal != 0x60 )) // $, @, `
194      || (UcnVal >= 0xD800 && UcnVal <= 0xDFFF)
195      || (UcnVal > 0x10FFFF)) /* the maximum legal UTF32 value */ {
196    PP.Diag(Loc, diag::err_ucn_escape_invalid);
197    HadError = 1;
198    return;
199  }
200  // Now that we've parsed/checked the UCN, we convert from UTF32->UTF8.
201  // The conversion below was inspired by:
202  //   http://www.unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c
203  // First, we determine how many bytes the result will require.
204  typedef uint8_t UTF8;
205
206  unsigned short bytesToWrite = 0;
207  if (UcnVal < (UTF32)0x80)
208    bytesToWrite = 1;
209  else if (UcnVal < (UTF32)0x800)
210    bytesToWrite = 2;
211  else if (UcnVal < (UTF32)0x10000)
212    bytesToWrite = 3;
213  else
214    bytesToWrite = 4;
215
216  const unsigned byteMask = 0xBF;
217  const unsigned byteMark = 0x80;
218
219  // Once the bits are split out into bytes of UTF8, this is a mask OR-ed
220  // into the first byte, depending on how many bytes follow.
221  static const UTF8 firstByteMark[5] = {
222    0x00, 0x00, 0xC0, 0xE0, 0xF0
223  };
224  // Finally, we write the bytes into ResultBuf.
225  ResultBuf += bytesToWrite;
226  switch (bytesToWrite) { // note: everything falls through.
227    case 4: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
228    case 3: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
229    case 2: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
230    case 1: *--ResultBuf = (UTF8) (UcnVal | firstByteMark[bytesToWrite]);
231  }
232  // Update the buffer.
233  ResultBuf += bytesToWrite;
234}
235
236
237///       integer-constant: [C99 6.4.4.1]
238///         decimal-constant integer-suffix
239///         octal-constant integer-suffix
240///         hexadecimal-constant integer-suffix
241///       decimal-constant:
242///         nonzero-digit
243///         decimal-constant digit
244///       octal-constant:
245///         0
246///         octal-constant octal-digit
247///       hexadecimal-constant:
248///         hexadecimal-prefix hexadecimal-digit
249///         hexadecimal-constant hexadecimal-digit
250///       hexadecimal-prefix: one of
251///         0x 0X
252///       integer-suffix:
253///         unsigned-suffix [long-suffix]
254///         unsigned-suffix [long-long-suffix]
255///         long-suffix [unsigned-suffix]
256///         long-long-suffix [unsigned-sufix]
257///       nonzero-digit:
258///         1 2 3 4 5 6 7 8 9
259///       octal-digit:
260///         0 1 2 3 4 5 6 7
261///       hexadecimal-digit:
262///         0 1 2 3 4 5 6 7 8 9
263///         a b c d e f
264///         A B C D E F
265///       unsigned-suffix: one of
266///         u U
267///       long-suffix: one of
268///         l L
269///       long-long-suffix: one of
270///         ll LL
271///
272///       floating-constant: [C99 6.4.4.2]
273///         TODO: add rules...
274///
275NumericLiteralParser::
276NumericLiteralParser(const char *begin, const char *end,
277                     SourceLocation TokLoc, Preprocessor &pp)
278  : PP(pp), ThisTokBegin(begin), ThisTokEnd(end) {
279
280  // This routine assumes that the range begin/end matches the regex for integer
281  // and FP constants (specifically, the 'pp-number' regex), and assumes that
282  // the byte at "*end" is both valid and not part of the regex.  Because of
283  // this, it doesn't have to check for 'overscan' in various places.
284  assert(!isalnum(*end) && *end != '.' && *end != '_' &&
285         "Lexer didn't maximally munch?");
286
287  s = DigitsBegin = begin;
288  saw_exponent = false;
289  saw_period = false;
290  isLong = false;
291  isUnsigned = false;
292  isLongLong = false;
293  isFloat = false;
294  isImaginary = false;
295  hadError = false;
296
297  if (*s == '0') { // parse radix
298    ParseNumberStartingWithZero(TokLoc);
299    if (hadError)
300      return;
301  } else { // the first digit is non-zero
302    radix = 10;
303    s = SkipDigits(s);
304    if (s == ThisTokEnd) {
305      // Done.
306    } else if (isxdigit(*s) && !(*s == 'e' || *s == 'E')) {
307      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-begin),
308              diag::err_invalid_decimal_digit) << std::string(s, s+1);
309      hadError = true;
310      return;
311    } else if (*s == '.') {
312      s++;
313      saw_period = true;
314      s = SkipDigits(s);
315    }
316    if ((*s == 'e' || *s == 'E')) { // exponent
317      const char *Exponent = s;
318      s++;
319      saw_exponent = true;
320      if (*s == '+' || *s == '-')  s++; // sign
321      const char *first_non_digit = SkipDigits(s);
322      if (first_non_digit != s) {
323        s = first_non_digit;
324      } else {
325        PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-begin),
326                diag::err_exponent_has_no_digits);
327        hadError = true;
328        return;
329      }
330    }
331  }
332
333  SuffixBegin = s;
334
335  // Parse the suffix.  At this point we can classify whether we have an FP or
336  // integer constant.
337  bool isFPConstant = isFloatingLiteral();
338
339  // Loop over all of the characters of the suffix.  If we see something bad,
340  // we break out of the loop.
341  for (; s != ThisTokEnd; ++s) {
342    switch (*s) {
343    case 'f':      // FP Suffix for "float"
344    case 'F':
345      if (!isFPConstant) break;  // Error for integer constant.
346      if (isFloat || isLong) break; // FF, LF invalid.
347      isFloat = true;
348      continue;  // Success.
349    case 'u':
350    case 'U':
351      if (isFPConstant) break;  // Error for floating constant.
352      if (isUnsigned) break;    // Cannot be repeated.
353      isUnsigned = true;
354      continue;  // Success.
355    case 'l':
356    case 'L':
357      if (isLong || isLongLong) break;  // Cannot be repeated.
358      if (isFloat) break;               // LF invalid.
359
360      // Check for long long.  The L's need to be adjacent and the same case.
361      if (s+1 != ThisTokEnd && s[1] == s[0]) {
362        if (isFPConstant) break;        // long long invalid for floats.
363        isLongLong = true;
364        ++s;  // Eat both of them.
365      } else {
366        isLong = true;
367      }
368      continue;  // Success.
369    case 'i':
370      if (PP.getLangOptions().Microsoft) {
371        // Allow i8, i16, i32, i64, and i128.
372        if (++s == ThisTokEnd) break;
373        switch (*s) {
374          case '8':
375            s++; // i8 suffix
376            break;
377          case '1':
378            if (++s == ThisTokEnd) break;
379            if (*s == '6') s++; // i16 suffix
380            else if (*s == '2') {
381              if (++s == ThisTokEnd) break;
382              if (*s == '8') s++; // i128 suffix
383            }
384            break;
385          case '3':
386            if (++s == ThisTokEnd) break;
387            if (*s == '2') s++; // i32 suffix
388            break;
389          case '6':
390            if (++s == ThisTokEnd) break;
391            if (*s == '4') s++; // i64 suffix
392            break;
393          default:
394            break;
395        }
396        break;
397      }
398      // fall through.
399    case 'I':
400    case 'j':
401    case 'J':
402      if (isImaginary) break;   // Cannot be repeated.
403      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-begin),
404              diag::ext_imaginary_constant);
405      isImaginary = true;
406      continue;  // Success.
407    }
408    // If we reached here, there was an error.
409    break;
410  }
411
412  // Report an error if there are any.
413  if (s != ThisTokEnd) {
414    PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-begin),
415            isFPConstant ? diag::err_invalid_suffix_float_constant :
416                           diag::err_invalid_suffix_integer_constant)
417      << std::string(SuffixBegin, ThisTokEnd);
418    hadError = true;
419    return;
420  }
421}
422
423/// ParseNumberStartingWithZero - This method is called when the first character
424/// of the number is found to be a zero.  This means it is either an octal
425/// number (like '04') or a hex number ('0x123a') a binary number ('0b1010') or
426/// a floating point number (01239.123e4).  Eat the prefix, determining the
427/// radix etc.
428void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
429  assert(s[0] == '0' && "Invalid method call");
430  s++;
431
432  // Handle a hex number like 0x1234.
433  if ((*s == 'x' || *s == 'X') && (isxdigit(s[1]) || s[1] == '.')) {
434    s++;
435    radix = 16;
436    DigitsBegin = s;
437    s = SkipHexDigits(s);
438    if (s == ThisTokEnd) {
439      // Done.
440    } else if (*s == '.') {
441      s++;
442      saw_period = true;
443      s = SkipHexDigits(s);
444    }
445    // A binary exponent can appear with or with a '.'. If dotted, the
446    // binary exponent is required.
447    if (*s == 'p' || *s == 'P') {
448      const char *Exponent = s;
449      s++;
450      saw_exponent = true;
451      if (*s == '+' || *s == '-')  s++; // sign
452      const char *first_non_digit = SkipDigits(s);
453      if (first_non_digit == s) {
454        PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-ThisTokBegin),
455                diag::err_exponent_has_no_digits);
456        hadError = true;
457        return;
458      }
459      s = first_non_digit;
460
461      if (!PP.getLangOptions().HexFloats)
462        PP.Diag(TokLoc, diag::ext_hexconstant_invalid);
463    } else if (saw_period) {
464      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin),
465              diag::err_hexconstant_requires_exponent);
466      hadError = true;
467    }
468    return;
469  }
470
471  // Handle simple binary numbers 0b01010
472  if (*s == 'b' || *s == 'B') {
473    // 0b101010 is a GCC extension.
474    PP.Diag(TokLoc, diag::ext_binary_literal);
475    ++s;
476    radix = 2;
477    DigitsBegin = s;
478    s = SkipBinaryDigits(s);
479    if (s == ThisTokEnd) {
480      // Done.
481    } else if (isxdigit(*s)) {
482      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin),
483              diag::err_invalid_binary_digit) << std::string(s, s+1);
484      hadError = true;
485    }
486    // Other suffixes will be diagnosed by the caller.
487    return;
488  }
489
490  // For now, the radix is set to 8. If we discover that we have a
491  // floating point constant, the radix will change to 10. Octal floating
492  // point constants are not permitted (only decimal and hexadecimal).
493  radix = 8;
494  DigitsBegin = s;
495  s = SkipOctalDigits(s);
496  if (s == ThisTokEnd)
497    return; // Done, simple octal number like 01234
498
499  // If we have some other non-octal digit that *is* a decimal digit, see if
500  // this is part of a floating point number like 094.123 or 09e1.
501  if (isdigit(*s)) {
502    const char *EndDecimal = SkipDigits(s);
503    if (EndDecimal[0] == '.' || EndDecimal[0] == 'e' || EndDecimal[0] == 'E') {
504      s = EndDecimal;
505      radix = 10;
506    }
507  }
508
509  // If we have a hex digit other than 'e' (which denotes a FP exponent) then
510  // the code is using an incorrect base.
511  if (isxdigit(*s) && *s != 'e' && *s != 'E') {
512    PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin),
513            diag::err_invalid_octal_digit) << std::string(s, s+1);
514    hadError = true;
515    return;
516  }
517
518  if (*s == '.') {
519    s++;
520    radix = 10;
521    saw_period = true;
522    s = SkipDigits(s); // Skip suffix.
523  }
524  if (*s == 'e' || *s == 'E') { // exponent
525    const char *Exponent = s;
526    s++;
527    radix = 10;
528    saw_exponent = true;
529    if (*s == '+' || *s == '-')  s++; // sign
530    const char *first_non_digit = SkipDigits(s);
531    if (first_non_digit != s) {
532      s = first_non_digit;
533    } else {
534      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-ThisTokBegin),
535              diag::err_exponent_has_no_digits);
536      hadError = true;
537      return;
538    }
539  }
540}
541
542
543/// GetIntegerValue - Convert this numeric literal value to an APInt that
544/// matches Val's input width.  If there is an overflow, set Val to the low bits
545/// of the result and return true.  Otherwise, return false.
546bool NumericLiteralParser::GetIntegerValue(llvm::APInt &Val) {
547  // Fast path: Compute a conservative bound on the maximum number of
548  // bits per digit in this radix. If we can't possibly overflow a
549  // uint64 based on that bound then do the simple conversion to
550  // integer. This avoids the expensive overflow checking below, and
551  // handles the common cases that matter (small decimal integers and
552  // hex/octal values which don't overflow).
553  unsigned MaxBitsPerDigit = 1;
554  while ((1U << MaxBitsPerDigit) < radix)
555    MaxBitsPerDigit += 1;
556  if ((SuffixBegin - DigitsBegin) * MaxBitsPerDigit <= 64) {
557    uint64_t N = 0;
558    for (s = DigitsBegin; s != SuffixBegin; ++s)
559      N = N*radix + HexDigitValue(*s);
560
561    // This will truncate the value to Val's input width. Simply check
562    // for overflow by comparing.
563    Val = N;
564    return Val.getZExtValue() != N;
565  }
566
567  Val = 0;
568  s = DigitsBegin;
569
570  llvm::APInt RadixVal(Val.getBitWidth(), radix);
571  llvm::APInt CharVal(Val.getBitWidth(), 0);
572  llvm::APInt OldVal = Val;
573
574  bool OverflowOccurred = false;
575  while (s < SuffixBegin) {
576    unsigned C = HexDigitValue(*s++);
577
578    // If this letter is out of bound for this radix, reject it.
579    assert(C < radix && "NumericLiteralParser ctor should have rejected this");
580
581    CharVal = C;
582
583    // Add the digit to the value in the appropriate radix.  If adding in digits
584    // made the value smaller, then this overflowed.
585    OldVal = Val;
586
587    // Multiply by radix, did overflow occur on the multiply?
588    Val *= RadixVal;
589    OverflowOccurred |= Val.udiv(RadixVal) != OldVal;
590
591    // Add value, did overflow occur on the value?
592    //   (a + b) ult b  <=> overflow
593    Val += CharVal;
594    OverflowOccurred |= Val.ult(CharVal);
595  }
596  return OverflowOccurred;
597}
598
599llvm::APFloat NumericLiteralParser::
600GetFloatValue(const llvm::fltSemantics &Format, bool* isExact) {
601  using llvm::APFloat;
602
603  llvm::SmallVector<char,256> floatChars;
604  for (unsigned i = 0, n = ThisTokEnd-ThisTokBegin; i != n; ++i)
605    floatChars.push_back(ThisTokBegin[i]);
606
607  floatChars.push_back('\0');
608
609  APFloat V (Format, APFloat::fcZero, false);
610  APFloat::opStatus status;
611
612  status = V.convertFromString(&floatChars[0],APFloat::rmNearestTiesToEven);
613
614  if (isExact)
615    *isExact = status == APFloat::opOK;
616
617  return V;
618}
619
620
621CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
622                                     SourceLocation Loc, Preprocessor &PP) {
623  // At this point we know that the character matches the regex "L?'.*'".
624  HadError = false;
625  Value = 0;
626
627  // Determine if this is a wide character.
628  IsWide = begin[0] == 'L';
629  if (IsWide) ++begin;
630
631  // Skip over the entry quote.
632  assert(begin[0] == '\'' && "Invalid token lexed");
633  ++begin;
634
635  // FIXME: This assumes that 'int' is 32-bits in overflow calculation, and the
636  // size of "value".
637  assert(PP.getTargetInfo().getIntWidth() == 32 &&
638         "Assumes sizeof(int) == 4 for now");
639  // FIXME: This assumes that wchar_t is 32-bits for now.
640  assert(PP.getTargetInfo().getWCharWidth() == 32 &&
641         "Assumes sizeof(wchar_t) == 4 for now");
642  // FIXME: This extensively assumes that 'char' is 8-bits.
643  assert(PP.getTargetInfo().getCharWidth() == 8 &&
644         "Assumes char is 8 bits");
645
646  bool isFirstChar = true;
647  bool isMultiChar = false;
648  while (begin[0] != '\'') {
649    unsigned ResultChar;
650    if (begin[0] != '\\')     // If this is a normal character, consume it.
651      ResultChar = *begin++;
652    else                      // Otherwise, this is an escape character.
653      ResultChar = ProcessCharEscape(begin, end, HadError, Loc, IsWide, PP);
654
655    // If this is a multi-character constant (e.g. 'abc'), handle it.  These are
656    // implementation defined (C99 6.4.4.4p10).
657    if (!isFirstChar) {
658      // If this is the second character being processed, do special handling.
659      if (!isMultiChar) {
660        isMultiChar = true;
661
662        // Warn about discarding the top bits for multi-char wide-character
663        // constants (L'abcd').
664        if (IsWide)
665          PP.Diag(Loc, diag::warn_extraneous_wide_char_constant);
666      }
667
668      if (IsWide) {
669        // Emulate GCC's (unintentional?) behavior: L'ab' -> L'b'.
670        Value = 0;
671      } else {
672        // Narrow character literals act as though their value is concatenated
673        // in this implementation.
674        if (((Value << 8) >> 8) != Value)
675          PP.Diag(Loc, diag::warn_char_constant_too_large);
676        Value <<= 8;
677      }
678    }
679
680    Value += ResultChar;
681    isFirstChar = false;
682  }
683
684  // If this is a single narrow character, sign extend it (e.g. '\xFF' is "-1")
685  // if 'char' is signed for this target (C99 6.4.4.4p10).  Note that multiple
686  // character constants are not sign extended in the this implementation:
687  // '\xFF\xFF' = 65536 and '\x0\xFF' = 255, which matches GCC.
688  if (!IsWide && !isMultiChar && (Value & 128) &&
689      PP.getTargetInfo().isCharSigned())
690    Value = (signed char)Value;
691}
692
693
694///       string-literal: [C99 6.4.5]
695///          " [s-char-sequence] "
696///         L" [s-char-sequence] "
697///       s-char-sequence:
698///         s-char
699///         s-char-sequence s-char
700///       s-char:
701///         any source character except the double quote ",
702///           backslash \, or newline character
703///         escape-character
704///         universal-character-name
705///       escape-character: [C99 6.4.4.4]
706///         \ escape-code
707///         universal-character-name
708///       escape-code:
709///         character-escape-code
710///         octal-escape-code
711///         hex-escape-code
712///       character-escape-code: one of
713///         n t b r f v a
714///         \ ' " ?
715///       octal-escape-code:
716///         octal-digit
717///         octal-digit octal-digit
718///         octal-digit octal-digit octal-digit
719///       hex-escape-code:
720///         x hex-digit
721///         hex-escape-code hex-digit
722///       universal-character-name:
723///         \u hex-quad
724///         \U hex-quad hex-quad
725///       hex-quad:
726///         hex-digit hex-digit hex-digit hex-digit
727///
728StringLiteralParser::
729StringLiteralParser(const Token *StringToks, unsigned NumStringToks,
730                    Preprocessor &pp) : PP(pp) {
731  // Scan all of the string portions, remember the max individual token length,
732  // computing a bound on the concatenated string length, and see whether any
733  // piece is a wide-string.  If any of the string portions is a wide-string
734  // literal, the result is a wide-string literal [C99 6.4.5p4].
735  MaxTokenLength = StringToks[0].getLength();
736  SizeBound = StringToks[0].getLength()-2;  // -2 for "".
737  AnyWide = StringToks[0].is(tok::wide_string_literal);
738
739  hadError = false;
740
741  // Implement Translation Phase #6: concatenation of string literals
742  /// (C99 5.1.1.2p1).  The common case is only one string fragment.
743  for (unsigned i = 1; i != NumStringToks; ++i) {
744    // The string could be shorter than this if it needs cleaning, but this is a
745    // reasonable bound, which is all we need.
746    SizeBound += StringToks[i].getLength()-2;  // -2 for "".
747
748    // Remember maximum string piece length.
749    if (StringToks[i].getLength() > MaxTokenLength)
750      MaxTokenLength = StringToks[i].getLength();
751
752    // Remember if we see any wide strings.
753    AnyWide |= StringToks[i].is(tok::wide_string_literal);
754  }
755
756  // Include space for the null terminator.
757  ++SizeBound;
758
759  // TODO: K&R warning: "traditional C rejects string constant concatenation"
760
761  // Get the width in bytes of wchar_t.  If no wchar_t strings are used, do not
762  // query the target.  As such, wchar_tByteWidth is only valid if AnyWide=true.
763  wchar_tByteWidth = ~0U;
764  if (AnyWide) {
765    wchar_tByteWidth = PP.getTargetInfo().getWCharWidth();
766    assert((wchar_tByteWidth & 7) == 0 && "Assumes wchar_t is byte multiple!");
767    wchar_tByteWidth /= 8;
768  }
769
770  // The output buffer size needs to be large enough to hold wide characters.
771  // This is a worst-case assumption which basically corresponds to L"" "long".
772  if (AnyWide)
773    SizeBound *= wchar_tByteWidth;
774
775  // Size the temporary buffer to hold the result string data.
776  ResultBuf.resize(SizeBound);
777
778  // Likewise, but for each string piece.
779  llvm::SmallString<512> TokenBuf;
780  TokenBuf.resize(MaxTokenLength);
781
782  // Loop over all the strings, getting their spelling, and expanding them to
783  // wide strings as appropriate.
784  ResultPtr = &ResultBuf[0];   // Next byte to fill in.
785
786  Pascal = false;
787
788  for (unsigned i = 0, e = NumStringToks; i != e; ++i) {
789    const char *ThisTokBuf = &TokenBuf[0];
790    // Get the spelling of the token, which eliminates trigraphs, etc.  We know
791    // that ThisTokBuf points to a buffer that is big enough for the whole token
792    // and 'spelled' tokens can only shrink.
793    unsigned ThisTokLen = PP.getSpelling(StringToks[i], ThisTokBuf);
794    const char *ThisTokEnd = ThisTokBuf+ThisTokLen-1;  // Skip end quote.
795
796    // TODO: Input character set mapping support.
797
798    // Skip L marker for wide strings.
799    bool ThisIsWide = false;
800    if (ThisTokBuf[0] == 'L') {
801      ++ThisTokBuf;
802      ThisIsWide = true;
803    }
804
805    assert(ThisTokBuf[0] == '"' && "Expected quote, lexer broken?");
806    ++ThisTokBuf;
807
808    // Check if this is a pascal string
809    if (pp.getLangOptions().PascalStrings && ThisTokBuf + 1 != ThisTokEnd &&
810        ThisTokBuf[0] == '\\' && ThisTokBuf[1] == 'p') {
811
812      // If the \p sequence is found in the first token, we have a pascal string
813      // Otherwise, if we already have a pascal string, ignore the first \p
814      if (i == 0) {
815        ++ThisTokBuf;
816        Pascal = true;
817      } else if (Pascal)
818        ThisTokBuf += 2;
819    }
820
821    while (ThisTokBuf != ThisTokEnd) {
822      // Is this a span of non-escape characters?
823      if (ThisTokBuf[0] != '\\') {
824        const char *InStart = ThisTokBuf;
825        do {
826          ++ThisTokBuf;
827        } while (ThisTokBuf != ThisTokEnd && ThisTokBuf[0] != '\\');
828
829        // Copy the character span over.
830        unsigned Len = ThisTokBuf-InStart;
831        if (!AnyWide) {
832          memcpy(ResultPtr, InStart, Len);
833          ResultPtr += Len;
834        } else {
835          // Note: our internal rep of wide char tokens is always little-endian.
836          for (; Len; --Len, ++InStart) {
837            *ResultPtr++ = InStart[0];
838            // Add zeros at the end.
839            for (unsigned i = 1, e = wchar_tByteWidth; i != e; ++i)
840              *ResultPtr++ = 0;
841          }
842        }
843        continue;
844      }
845      // Is this a Universal Character Name escape?
846      if (ThisTokBuf[1] == 'u' || ThisTokBuf[1] == 'U') {
847        ProcessUCNEscape(ThisTokBuf, ThisTokEnd, ResultPtr,
848                         hadError, StringToks[i].getLocation(), ThisIsWide, PP);
849        continue;
850      }
851      // Otherwise, this is a non-UCN escape character.  Process it.
852      unsigned ResultChar = ProcessCharEscape(ThisTokBuf, ThisTokEnd, hadError,
853                                              StringToks[i].getLocation(),
854                                              ThisIsWide, PP);
855
856      // Note: our internal rep of wide char tokens is always little-endian.
857      *ResultPtr++ = ResultChar & 0xFF;
858
859      if (AnyWide) {
860        for (unsigned i = 1, e = wchar_tByteWidth; i != e; ++i)
861          *ResultPtr++ = ResultChar >> i*8;
862      }
863    }
864  }
865
866  if (Pascal) {
867    ResultBuf[0] = ResultPtr-&ResultBuf[0]-1;
868
869    // Verify that pascal strings aren't too large.
870    if (GetStringLength() > 256) {
871      PP.Diag(StringToks[0].getLocation(), diag::err_pascal_string_too_long)
872        << SourceRange(StringToks[0].getLocation(),
873                       StringToks[NumStringToks-1].getLocation());
874      hadError = 1;
875      return;
876    }
877  }
878}
879
880
881/// getOffsetOfStringByte - This function returns the offset of the
882/// specified byte of the string data represented by Token.  This handles
883/// advancing over escape sequences in the string.
884unsigned StringLiteralParser::getOffsetOfStringByte(const Token &Tok,
885                                                    unsigned ByteNo,
886                                                    Preprocessor &PP) {
887  // Get the spelling of the token.
888  llvm::SmallString<16> SpellingBuffer;
889  SpellingBuffer.resize(Tok.getLength());
890
891  const char *SpellingPtr = &SpellingBuffer[0];
892  unsigned TokLen = PP.getSpelling(Tok, SpellingPtr);
893
894  assert(SpellingPtr[0] != 'L' && "Doesn't handle wide strings yet");
895
896
897  const char *SpellingStart = SpellingPtr;
898  const char *SpellingEnd = SpellingPtr+TokLen;
899
900  // Skip over the leading quote.
901  assert(SpellingPtr[0] == '"' && "Should be a string literal!");
902  ++SpellingPtr;
903
904  // Skip over bytes until we find the offset we're looking for.
905  while (ByteNo) {
906    assert(SpellingPtr < SpellingEnd && "Didn't find byte offset!");
907
908    // Step over non-escapes simply.
909    if (*SpellingPtr != '\\') {
910      ++SpellingPtr;
911      --ByteNo;
912      continue;
913    }
914
915    // Otherwise, this is an escape character.  Advance over it.
916    bool HadError = false;
917    ProcessCharEscape(SpellingPtr, SpellingEnd, HadError,
918                      Tok.getLocation(), false, PP);
919    assert(!HadError && "This method isn't valid on erroneous strings");
920    --ByteNo;
921  }
922
923  return SpellingPtr-SpellingStart;
924}
925