LiteralSupport.cpp revision 476d8b863cb65b2b5833235d97315cdb46e6f5aa 
1//===--- LiteralSupport.cpp - Code to parse and process literals ----------===//
2//
3//                     The LLVM Compiler Infrastructure
4//
5// This file is distributed under the University of Illinois Open Source
6// License. See LICENSE.TXT for details.
7//
8//===----------------------------------------------------------------------===//
9//
10// This file implements the NumericLiteralParser, CharLiteralParser, and
11// StringLiteralParser interfaces.
12//
13//===----------------------------------------------------------------------===//
14
15#include "clang/Lex/LiteralSupport.h"
16#include "clang/Lex/Preprocessor.h"
17#include "clang/Lex/LexDiagnostic.h"
18#include "clang/Basic/TargetInfo.h"
19#include "llvm/ADT/StringRef.h"
20#include "llvm/ADT/StringExtras.h"
21using namespace clang;
22
23/// HexDigitValue - Return the value of the specified hex digit, or -1 if it's
24/// not valid.
25static int HexDigitValue(char C) {
26  if (C >= '0' && C <= '9') return C-'0';
27  if (C >= 'a' && C <= 'f') return C-'a'+10;
28  if (C >= 'A' && C <= 'F') return C-'A'+10;
29  return -1;
30}
31
32/// ProcessCharEscape - Parse a standard C escape sequence, which can occur in
33/// either a character or a string literal.
34static unsigned ProcessCharEscape(const char *&ThisTokBuf,
35                                  const char *ThisTokEnd, bool &HadError,
36                                  SourceLocation Loc, bool IsWide,
37                                  Preprocessor &PP, bool Complain) {
38  // Skip the '\' char.
39  ++ThisTokBuf;
40
41  // We know that this character can't be off the end of the buffer, because
42  // that would have been \", which would not have been the end of string.
43  unsigned ResultChar = *ThisTokBuf++;
44  switch (ResultChar) {
45  // These map to themselves.
46  case '\\': case '\'': case '"': case '?': break;
47
48    // These have fixed mappings.
49  case 'a':
50    // TODO: K&R: the meaning of '\\a' is different in traditional C
51    ResultChar = 7;
52    break;
53  case 'b':
54    ResultChar = 8;
55    break;
56  case 'e':
57    if (Complain)
58      PP.Diag(Loc, diag::ext_nonstandard_escape) << "e";
59    ResultChar = 27;
60    break;
61  case 'E':
62    if (Complain)
63      PP.Diag(Loc, diag::ext_nonstandard_escape) << "E";
64    ResultChar = 27;
65    break;
66  case 'f':
67    ResultChar = 12;
68    break;
69  case 'n':
70    ResultChar = 10;
71    break;
72  case 'r':
73    ResultChar = 13;
74    break;
75  case 't':
76    ResultChar = 9;
77    break;
78  case 'v':
79    ResultChar = 11;
80    break;
81  case 'x': { // Hex escape.
82    ResultChar = 0;
83    if (ThisTokBuf == ThisTokEnd || !isxdigit(*ThisTokBuf)) {
84      if (Complain)
85        PP.Diag(Loc, diag::err_hex_escape_no_digits);
86      HadError = 1;
87      break;
88    }
89
90    // Hex escapes are a maximal series of hex digits.
91    bool Overflow = false;
92    for (; ThisTokBuf != ThisTokEnd; ++ThisTokBuf) {
93      int CharVal = HexDigitValue(ThisTokBuf[0]);
94      if (CharVal == -1) break;
95      // About to shift out a digit?
96      Overflow |= (ResultChar & 0xF0000000) ? true : false;
97      ResultChar <<= 4;
98      ResultChar |= CharVal;
99    }
100
101    // See if any bits will be truncated when evaluated as a character.
102    unsigned CharWidth = IsWide
103                       ? PP.getTargetInfo().getWCharWidth()
104                       : PP.getTargetInfo().getCharWidth();
105
106    if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
107      Overflow = true;
108      ResultChar &= ~0U >> (32-CharWidth);
109    }
110
111    // Check for overflow.
112    if (Overflow && Complain)   // Too many digits to fit in
113      PP.Diag(Loc, diag::warn_hex_escape_too_large);
114    break;
115  }
116  case '0': case '1': case '2': case '3':
117  case '4': case '5': case '6': case '7': {
118    // Octal escapes.
119    --ThisTokBuf;
120    ResultChar = 0;
121
122    // Octal escapes are a series of octal digits with maximum length 3.
123    // "\0123" is a two digit sequence equal to "\012" "3".
124    unsigned NumDigits = 0;
125    do {
126      ResultChar <<= 3;
127      ResultChar |= *ThisTokBuf++ - '0';
128      ++NumDigits;
129    } while (ThisTokBuf != ThisTokEnd && NumDigits < 3 &&
130             ThisTokBuf[0] >= '0' && ThisTokBuf[0] <= '7');
131
132    // Check for overflow.  Reject '\777', but not L'\777'.
133    unsigned CharWidth = IsWide
134                       ? PP.getTargetInfo().getWCharWidth()
135                       : PP.getTargetInfo().getCharWidth();
136
137    if (CharWidth != 32 && (ResultChar >> CharWidth) != 0) {
138      if (Complain)
139        PP.Diag(Loc, diag::warn_octal_escape_too_large);
140      ResultChar &= ~0U >> (32-CharWidth);
141    }
142    break;
143  }
144
145    // Otherwise, these are not valid escapes.
146  case '(': case '{': case '[': case '%':
147    // GCC accepts these as extensions.  We warn about them as such though.
148    if (Complain)
149      PP.Diag(Loc, diag::ext_nonstandard_escape)
150        << std::string()+(char)ResultChar;
151    break;
152  default:
153    if (!Complain)
154      break;
155
156    if (isgraph(ThisTokBuf[0]))
157      PP.Diag(Loc, diag::ext_unknown_escape) << std::string()+(char)ResultChar;
158    else
159      PP.Diag(Loc, diag::ext_unknown_escape) << "x"+llvm::utohexstr(ResultChar);
160    break;
161  }
162
163  return ResultChar;
164}
165
166/// ProcessUCNEscape - Read the Universal Character Name, check constraints and
167/// convert the UTF32 to UTF8. This is a subroutine of StringLiteralParser.
168/// When we decide to implement UCN's for character constants and identifiers,
169/// we will likely rework our support for UCN's.
170static void ProcessUCNEscape(const char *&ThisTokBuf, const char *ThisTokEnd,
171                             char *&ResultBuf, bool &HadError,
172                             SourceLocation Loc, Preprocessor &PP,
173                             bool Complain) {
174  // FIXME: Add a warning - UCN's are only valid in C++ & C99.
175  // FIXME: Handle wide strings.
176
177  // Save the beginning of the string (for error diagnostics).
178  const char *ThisTokBegin = ThisTokBuf;
179
180  // Skip the '\u' char's.
181  ThisTokBuf += 2;
182
183  if (ThisTokBuf == ThisTokEnd || !isxdigit(*ThisTokBuf)) {
184    if (Complain)
185      PP.Diag(Loc, diag::err_ucn_escape_no_digits);
186    HadError = 1;
187    return;
188  }
189  typedef uint32_t UTF32;
190
191  UTF32 UcnVal = 0;
192  unsigned short UcnLen = (ThisTokBuf[-1] == 'u' ? 4 : 8);
193  for (; ThisTokBuf != ThisTokEnd && UcnLen; ++ThisTokBuf, UcnLen--) {
194    int CharVal = HexDigitValue(ThisTokBuf[0]);
195    if (CharVal == -1) break;
196    UcnVal <<= 4;
197    UcnVal |= CharVal;
198  }
199  // If we didn't consume the proper number of digits, there is a problem.
200  if (UcnLen) {
201    if (Complain)
202      PP.Diag(PP.AdvanceToTokenCharacter(Loc, ThisTokBuf-ThisTokBegin),
203              diag::err_ucn_escape_incomplete);
204    HadError = 1;
205    return;
206  }
207  // Check UCN constraints (C99 6.4.3p2).
208  if ((UcnVal < 0xa0 &&
209      (UcnVal != 0x24 && UcnVal != 0x40 && UcnVal != 0x60 )) // $, @, `
210      || (UcnVal >= 0xD800 && UcnVal <= 0xDFFF)
211      || (UcnVal > 0x10FFFF)) /* the maximum legal UTF32 value */ {
212    if (Complain)
213      PP.Diag(Loc, diag::err_ucn_escape_invalid);
214    HadError = 1;
215    return;
216  }
217  // Now that we've parsed/checked the UCN, we convert from UTF32->UTF8.
218  // The conversion below was inspired by:
219  //   http://www.unicode.org/Public/PROGRAMS/CVTUTF/ConvertUTF.c
220  // First, we determine how many bytes the result will require.
221  typedef uint8_t UTF8;
222
223  unsigned short bytesToWrite = 0;
224  if (UcnVal < (UTF32)0x80)
225    bytesToWrite = 1;
226  else if (UcnVal < (UTF32)0x800)
227    bytesToWrite = 2;
228  else if (UcnVal < (UTF32)0x10000)
229    bytesToWrite = 3;
230  else
231    bytesToWrite = 4;
232
233  const unsigned byteMask = 0xBF;
234  const unsigned byteMark = 0x80;
235
236  // Once the bits are split out into bytes of UTF8, this is a mask OR-ed
237  // into the first byte, depending on how many bytes follow.
238  static const UTF8 firstByteMark[5] = {
239    0x00, 0x00, 0xC0, 0xE0, 0xF0
240  };
241  // Finally, we write the bytes into ResultBuf.
242  ResultBuf += bytesToWrite;
243  switch (bytesToWrite) { // note: everything falls through.
244    case 4: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
245    case 3: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
246    case 2: *--ResultBuf = (UTF8)((UcnVal | byteMark) & byteMask); UcnVal >>= 6;
247    case 1: *--ResultBuf = (UTF8) (UcnVal | firstByteMark[bytesToWrite]);
248  }
249  // Update the buffer.
250  ResultBuf += bytesToWrite;
251}
252
253
254///       integer-constant: [C99 6.4.4.1]
255///         decimal-constant integer-suffix
256///         octal-constant integer-suffix
257///         hexadecimal-constant integer-suffix
258///       decimal-constant:
259///         nonzero-digit
260///         decimal-constant digit
261///       octal-constant:
262///         0
263///         octal-constant octal-digit
264///       hexadecimal-constant:
265///         hexadecimal-prefix hexadecimal-digit
266///         hexadecimal-constant hexadecimal-digit
267///       hexadecimal-prefix: one of
268///         0x 0X
269///       integer-suffix:
270///         unsigned-suffix [long-suffix]
271///         unsigned-suffix [long-long-suffix]
272///         long-suffix [unsigned-suffix]
273///         long-long-suffix [unsigned-sufix]
274///       nonzero-digit:
275///         1 2 3 4 5 6 7 8 9
276///       octal-digit:
277///         0 1 2 3 4 5 6 7
278///       hexadecimal-digit:
279///         0 1 2 3 4 5 6 7 8 9
280///         a b c d e f
281///         A B C D E F
282///       unsigned-suffix: one of
283///         u U
284///       long-suffix: one of
285///         l L
286///       long-long-suffix: one of
287///         ll LL
288///
289///       floating-constant: [C99 6.4.4.2]
290///         TODO: add rules...
291///
292NumericLiteralParser::
293NumericLiteralParser(const char *begin, const char *end,
294                     SourceLocation TokLoc, Preprocessor &pp)
295  : PP(pp), ThisTokBegin(begin), ThisTokEnd(end) {
296
297  // This routine assumes that the range begin/end matches the regex for integer
298  // and FP constants (specifically, the 'pp-number' regex), and assumes that
299  // the byte at "*end" is both valid and not part of the regex.  Because of
300  // this, it doesn't have to check for 'overscan' in various places.
301  assert(!isalnum(*end) && *end != '.' && *end != '_' &&
302         "Lexer didn't maximally munch?");
303
304  s = DigitsBegin = begin;
305  saw_exponent = false;
306  saw_period = false;
307  isLong = false;
308  isUnsigned = false;
309  isLongLong = false;
310  isFloat = false;
311  isImaginary = false;
312  isMicrosoftInteger = false;
313  hadError = false;
314
315  if (*s == '0') { // parse radix
316    ParseNumberStartingWithZero(TokLoc);
317    if (hadError)
318      return;
319  } else { // the first digit is non-zero
320    radix = 10;
321    s = SkipDigits(s);
322    if (s == ThisTokEnd) {
323      // Done.
324    } else if (isxdigit(*s) && !(*s == 'e' || *s == 'E')) {
325      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-begin),
326              diag::err_invalid_decimal_digit) << llvm::StringRef(s, 1);
327      hadError = true;
328      return;
329    } else if (*s == '.') {
330      s++;
331      saw_period = true;
332      s = SkipDigits(s);
333    }
334    if ((*s == 'e' || *s == 'E')) { // exponent
335      const char *Exponent = s;
336      s++;
337      saw_exponent = true;
338      if (*s == '+' || *s == '-')  s++; // sign
339      const char *first_non_digit = SkipDigits(s);
340      if (first_non_digit != s) {
341        s = first_non_digit;
342      } else {
343        PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-begin),
344                diag::err_exponent_has_no_digits);
345        hadError = true;
346        return;
347      }
348    }
349  }
350
351  SuffixBegin = s;
352
353  // Parse the suffix.  At this point we can classify whether we have an FP or
354  // integer constant.
355  bool isFPConstant = isFloatingLiteral();
356
357  // Loop over all of the characters of the suffix.  If we see something bad,
358  // we break out of the loop.
359  for (; s != ThisTokEnd; ++s) {
360    switch (*s) {
361    case 'f':      // FP Suffix for "float"
362    case 'F':
363      if (!isFPConstant) break;  // Error for integer constant.
364      if (isFloat || isLong) break; // FF, LF invalid.
365      isFloat = true;
366      continue;  // Success.
367    case 'u':
368    case 'U':
369      if (isFPConstant) break;  // Error for floating constant.
370      if (isUnsigned) break;    // Cannot be repeated.
371      isUnsigned = true;
372      continue;  // Success.
373    case 'l':
374    case 'L':
375      if (isLong || isLongLong) break;  // Cannot be repeated.
376      if (isFloat) break;               // LF invalid.
377
378      // Check for long long.  The L's need to be adjacent and the same case.
379      if (s+1 != ThisTokEnd && s[1] == s[0]) {
380        if (isFPConstant) break;        // long long invalid for floats.
381        isLongLong = true;
382        ++s;  // Eat both of them.
383      } else {
384        isLong = true;
385      }
386      continue;  // Success.
387    case 'i':
388      if (PP.getLangOptions().Microsoft) {
389        if (isFPConstant || isLong || isLongLong) break;
390
391        // Allow i8, i16, i32, i64, and i128.
392        if (s + 1 != ThisTokEnd) {
393          switch (s[1]) {
394            case '8':
395              s += 2; // i8 suffix
396              isMicrosoftInteger = true;
397              break;
398            case '1':
399              if (s + 2 == ThisTokEnd) break;
400              if (s[2] == '6') s += 3; // i16 suffix
401              else if (s[2] == '2') {
402                if (s + 3 == ThisTokEnd) break;
403                if (s[3] == '8') s += 4; // i128 suffix
404              }
405              isMicrosoftInteger = true;
406              break;
407            case '3':
408              if (s + 2 == ThisTokEnd) break;
409              if (s[2] == '2') s += 3; // i32 suffix
410              isMicrosoftInteger = true;
411              break;
412            case '6':
413              if (s + 2 == ThisTokEnd) break;
414              if (s[2] == '4') s += 3; // i64 suffix
415              isMicrosoftInteger = true;
416              break;
417            default:
418              break;
419          }
420          break;
421        }
422      }
423      // fall through.
424    case 'I':
425    case 'j':
426    case 'J':
427      if (isImaginary) break;   // Cannot be repeated.
428      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-begin),
429              diag::ext_imaginary_constant);
430      isImaginary = true;
431      continue;  // Success.
432    }
433    // If we reached here, there was an error.
434    break;
435  }
436
437  // Report an error if there are any.
438  if (s != ThisTokEnd) {
439    PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-begin),
440            isFPConstant ? diag::err_invalid_suffix_float_constant :
441                           diag::err_invalid_suffix_integer_constant)
442      << llvm::StringRef(SuffixBegin, ThisTokEnd-SuffixBegin);
443    hadError = true;
444    return;
445  }
446}
447
448/// ParseNumberStartingWithZero - This method is called when the first character
449/// of the number is found to be a zero.  This means it is either an octal
450/// number (like '04') or a hex number ('0x123a') a binary number ('0b1010') or
451/// a floating point number (01239.123e4).  Eat the prefix, determining the
452/// radix etc.
453void NumericLiteralParser::ParseNumberStartingWithZero(SourceLocation TokLoc) {
454  assert(s[0] == '0' && "Invalid method call");
455  s++;
456
457  // Handle a hex number like 0x1234.
458  if ((*s == 'x' || *s == 'X') && (isxdigit(s[1]) || s[1] == '.')) {
459    s++;
460    radix = 16;
461    DigitsBegin = s;
462    s = SkipHexDigits(s);
463    if (s == ThisTokEnd) {
464      // Done.
465    } else if (*s == '.') {
466      s++;
467      saw_period = true;
468      s = SkipHexDigits(s);
469    }
470    // A binary exponent can appear with or with a '.'. If dotted, the
471    // binary exponent is required.
472    if ((*s == 'p' || *s == 'P') && !PP.getLangOptions().CPlusPlus0x) {
473      const char *Exponent = s;
474      s++;
475      saw_exponent = true;
476      if (*s == '+' || *s == '-')  s++; // sign
477      const char *first_non_digit = SkipDigits(s);
478      if (first_non_digit == s) {
479        PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-ThisTokBegin),
480                diag::err_exponent_has_no_digits);
481        hadError = true;
482        return;
483      }
484      s = first_non_digit;
485
486      // In C++0x, we cannot support hexadecmial floating literals because
487      // they conflict with user-defined literals, so we warn in previous
488      // versions of C++ by default.
489      if (PP.getLangOptions().CPlusPlus)
490        PP.Diag(TokLoc, diag::ext_hexconstant_cplusplus);
491      else if (!PP.getLangOptions().HexFloats)
492        PP.Diag(TokLoc, diag::ext_hexconstant_invalid);
493    } else if (saw_period) {
494      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin),
495              diag::err_hexconstant_requires_exponent);
496      hadError = true;
497    }
498    return;
499  }
500
501  // Handle simple binary numbers 0b01010
502  if (*s == 'b' || *s == 'B') {
503    // 0b101010 is a GCC extension.
504    PP.Diag(TokLoc, diag::ext_binary_literal);
505    ++s;
506    radix = 2;
507    DigitsBegin = s;
508    s = SkipBinaryDigits(s);
509    if (s == ThisTokEnd) {
510      // Done.
511    } else if (isxdigit(*s)) {
512      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin),
513              diag::err_invalid_binary_digit) << llvm::StringRef(s, 1);
514      hadError = true;
515    }
516    // Other suffixes will be diagnosed by the caller.
517    return;
518  }
519
520  // For now, the radix is set to 8. If we discover that we have a
521  // floating point constant, the radix will change to 10. Octal floating
522  // point constants are not permitted (only decimal and hexadecimal).
523  radix = 8;
524  DigitsBegin = s;
525  s = SkipOctalDigits(s);
526  if (s == ThisTokEnd)
527    return; // Done, simple octal number like 01234
528
529  // If we have some other non-octal digit that *is* a decimal digit, see if
530  // this is part of a floating point number like 094.123 or 09e1.
531  if (isdigit(*s)) {
532    const char *EndDecimal = SkipDigits(s);
533    if (EndDecimal[0] == '.' || EndDecimal[0] == 'e' || EndDecimal[0] == 'E') {
534      s = EndDecimal;
535      radix = 10;
536    }
537  }
538
539  // If we have a hex digit other than 'e' (which denotes a FP exponent) then
540  // the code is using an incorrect base.
541  if (isxdigit(*s) && *s != 'e' && *s != 'E') {
542    PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, s-ThisTokBegin),
543            diag::err_invalid_octal_digit) << llvm::StringRef(s, 1);
544    hadError = true;
545    return;
546  }
547
548  if (*s == '.') {
549    s++;
550    radix = 10;
551    saw_period = true;
552    s = SkipDigits(s); // Skip suffix.
553  }
554  if (*s == 'e' || *s == 'E') { // exponent
555    const char *Exponent = s;
556    s++;
557    radix = 10;
558    saw_exponent = true;
559    if (*s == '+' || *s == '-')  s++; // sign
560    const char *first_non_digit = SkipDigits(s);
561    if (first_non_digit != s) {
562      s = first_non_digit;
563    } else {
564      PP.Diag(PP.AdvanceToTokenCharacter(TokLoc, Exponent-ThisTokBegin),
565              diag::err_exponent_has_no_digits);
566      hadError = true;
567      return;
568    }
569  }
570}
571
572
573/// GetIntegerValue - Convert this numeric literal value to an APInt that
574/// matches Val's input width.  If there is an overflow, set Val to the low bits
575/// of the result and return true.  Otherwise, return false.
576bool NumericLiteralParser::GetIntegerValue(llvm::APInt &Val) {
577  // Fast path: Compute a conservative bound on the maximum number of
578  // bits per digit in this radix. If we can't possibly overflow a
579  // uint64 based on that bound then do the simple conversion to
580  // integer. This avoids the expensive overflow checking below, and
581  // handles the common cases that matter (small decimal integers and
582  // hex/octal values which don't overflow).
583  unsigned MaxBitsPerDigit = 1;
584  while ((1U << MaxBitsPerDigit) < radix)
585    MaxBitsPerDigit += 1;
586  if ((SuffixBegin - DigitsBegin) * MaxBitsPerDigit <= 64) {
587    uint64_t N = 0;
588    for (s = DigitsBegin; s != SuffixBegin; ++s)
589      N = N*radix + HexDigitValue(*s);
590
591    // This will truncate the value to Val's input width. Simply check
592    // for overflow by comparing.
593    Val = N;
594    return Val.getZExtValue() != N;
595  }
596
597  Val = 0;
598  s = DigitsBegin;
599
600  llvm::APInt RadixVal(Val.getBitWidth(), radix);
601  llvm::APInt CharVal(Val.getBitWidth(), 0);
602  llvm::APInt OldVal = Val;
603
604  bool OverflowOccurred = false;
605  while (s < SuffixBegin) {
606    unsigned C = HexDigitValue(*s++);
607
608    // If this letter is out of bound for this radix, reject it.
609    assert(C < radix && "NumericLiteralParser ctor should have rejected this");
610
611    CharVal = C;
612
613    // Add the digit to the value in the appropriate radix.  If adding in digits
614    // made the value smaller, then this overflowed.
615    OldVal = Val;
616
617    // Multiply by radix, did overflow occur on the multiply?
618    Val *= RadixVal;
619    OverflowOccurred |= Val.udiv(RadixVal) != OldVal;
620
621    // Add value, did overflow occur on the value?
622    //   (a + b) ult b  <=> overflow
623    Val += CharVal;
624    OverflowOccurred |= Val.ult(CharVal);
625  }
626  return OverflowOccurred;
627}
628
629llvm::APFloat::opStatus
630NumericLiteralParser::GetFloatValue(llvm::APFloat &Result) {
631  using llvm::APFloat;
632  using llvm::StringRef;
633
634  unsigned n = std::min(SuffixBegin - ThisTokBegin, ThisTokEnd - ThisTokBegin);
635  return Result.convertFromString(StringRef(ThisTokBegin, n),
636                                  APFloat::rmNearestTiesToEven);
637}
638
639
640CharLiteralParser::CharLiteralParser(const char *begin, const char *end,
641                                     SourceLocation Loc, Preprocessor &PP) {
642  // At this point we know that the character matches the regex "L?'.*'".
643  HadError = false;
644
645  // Determine if this is a wide character.
646  IsWide = begin[0] == 'L';
647  if (IsWide) ++begin;
648
649  // Skip over the entry quote.
650  assert(begin[0] == '\'' && "Invalid token lexed");
651  ++begin;
652
653  // FIXME: The "Value" is an uint64_t so we can handle char literals of
654  // upto 64-bits.
655  // FIXME: This extensively assumes that 'char' is 8-bits.
656  assert(PP.getTargetInfo().getCharWidth() == 8 &&
657         "Assumes char is 8 bits");
658  assert(PP.getTargetInfo().getIntWidth() <= 64 &&
659         (PP.getTargetInfo().getIntWidth() & 7) == 0 &&
660         "Assumes sizeof(int) on target is <= 64 and a multiple of char");
661  assert(PP.getTargetInfo().getWCharWidth() <= 64 &&
662         "Assumes sizeof(wchar) on target is <= 64");
663
664  // This is what we will use for overflow detection
665  llvm::APInt LitVal(PP.getTargetInfo().getIntWidth(), 0);
666
667  unsigned NumCharsSoFar = 0;
668  bool Warned = false;
669  while (begin[0] != '\'') {
670    uint64_t ResultChar;
671    if (begin[0] != '\\')     // If this is a normal character, consume it.
672      ResultChar = *begin++;
673    else                      // Otherwise, this is an escape character.
674      ResultChar = ProcessCharEscape(begin, end, HadError, Loc, IsWide, PP,
675                                     /*Complain=*/true);
676
677    // If this is a multi-character constant (e.g. 'abc'), handle it.  These are
678    // implementation defined (C99 6.4.4.4p10).
679    if (NumCharsSoFar) {
680      if (IsWide) {
681        // Emulate GCC's (unintentional?) behavior: L'ab' -> L'b'.
682        LitVal = 0;
683      } else {
684        // Narrow character literals act as though their value is concatenated
685        // in this implementation, but warn on overflow.
686        if (LitVal.countLeadingZeros() < 8 && !Warned) {
687          PP.Diag(Loc, diag::warn_char_constant_too_large);
688          Warned = true;
689        }
690        LitVal <<= 8;
691      }
692    }
693
694    LitVal = LitVal + ResultChar;
695    ++NumCharsSoFar;
696  }
697
698  // If this is the second character being processed, do special handling.
699  if (NumCharsSoFar > 1) {
700    // Warn about discarding the top bits for multi-char wide-character
701    // constants (L'abcd').
702    if (IsWide)
703      PP.Diag(Loc, diag::warn_extraneous_wide_char_constant);
704    else if (NumCharsSoFar != 4)
705      PP.Diag(Loc, diag::ext_multichar_character_literal);
706    else
707      PP.Diag(Loc, diag::ext_four_char_character_literal);
708    IsMultiChar = true;
709  } else
710    IsMultiChar = false;
711
712  // Transfer the value from APInt to uint64_t
713  Value = LitVal.getZExtValue();
714
715  // If this is a single narrow character, sign extend it (e.g. '\xFF' is "-1")
716  // if 'char' is signed for this target (C99 6.4.4.4p10).  Note that multiple
717  // character constants are not sign extended in the this implementation:
718  // '\xFF\xFF' = 65536 and '\x0\xFF' = 255, which matches GCC.
719  if (!IsWide && NumCharsSoFar == 1 && (Value & 128) &&
720      PP.getLangOptions().CharIsSigned)
721    Value = (signed char)Value;
722}
723
724
725///       string-literal: [C99 6.4.5]
726///          " [s-char-sequence] "
727///         L" [s-char-sequence] "
728///       s-char-sequence:
729///         s-char
730///         s-char-sequence s-char
731///       s-char:
732///         any source character except the double quote ",
733///           backslash \, or newline character
734///         escape-character
735///         universal-character-name
736///       escape-character: [C99 6.4.4.4]
737///         \ escape-code
738///         universal-character-name
739///       escape-code:
740///         character-escape-code
741///         octal-escape-code
742///         hex-escape-code
743///       character-escape-code: one of
744///         n t b r f v a
745///         \ ' " ?
746///       octal-escape-code:
747///         octal-digit
748///         octal-digit octal-digit
749///         octal-digit octal-digit octal-digit
750///       hex-escape-code:
751///         x hex-digit
752///         hex-escape-code hex-digit
753///       universal-character-name:
754///         \u hex-quad
755///         \U hex-quad hex-quad
756///       hex-quad:
757///         hex-digit hex-digit hex-digit hex-digit
758///
759StringLiteralParser::
760StringLiteralParser(const Token *StringToks, unsigned NumStringToks,
761                    Preprocessor &pp, bool Complain) : PP(pp) {
762  // Scan all of the string portions, remember the max individual token length,
763  // computing a bound on the concatenated string length, and see whether any
764  // piece is a wide-string.  If any of the string portions is a wide-string
765  // literal, the result is a wide-string literal [C99 6.4.5p4].
766  MaxTokenLength = StringToks[0].getLength();
767  SizeBound = StringToks[0].getLength()-2;  // -2 for "".
768  AnyWide = StringToks[0].is(tok::wide_string_literal);
769
770  hadError = false;
771
772  // Implement Translation Phase #6: concatenation of string literals
773  /// (C99 5.1.1.2p1).  The common case is only one string fragment.
774  for (unsigned i = 1; i != NumStringToks; ++i) {
775    // The string could be shorter than this if it needs cleaning, but this is a
776    // reasonable bound, which is all we need.
777    SizeBound += StringToks[i].getLength()-2;  // -2 for "".
778
779    // Remember maximum string piece length.
780    if (StringToks[i].getLength() > MaxTokenLength)
781      MaxTokenLength = StringToks[i].getLength();
782
783    // Remember if we see any wide strings.
784    AnyWide |= StringToks[i].is(tok::wide_string_literal);
785  }
786
787  // Include space for the null terminator.
788  ++SizeBound;
789
790  // TODO: K&R warning: "traditional C rejects string constant concatenation"
791
792  // Get the width in bytes of wchar_t.  If no wchar_t strings are used, do not
793  // query the target.  As such, wchar_tByteWidth is only valid if AnyWide=true.
794  wchar_tByteWidth = ~0U;
795  if (AnyWide) {
796    wchar_tByteWidth = PP.getTargetInfo().getWCharWidth();
797    assert((wchar_tByteWidth & 7) == 0 && "Assumes wchar_t is byte multiple!");
798    wchar_tByteWidth /= 8;
799  }
800
801  // The output buffer size needs to be large enough to hold wide characters.
802  // This is a worst-case assumption which basically corresponds to L"" "long".
803  if (AnyWide)
804    SizeBound *= wchar_tByteWidth;
805
806  // Size the temporary buffer to hold the result string data.
807  ResultBuf.resize(SizeBound);
808
809  // Likewise, but for each string piece.
810  llvm::SmallString<512> TokenBuf;
811  TokenBuf.resize(MaxTokenLength);
812
813  // Loop over all the strings, getting their spelling, and expanding them to
814  // wide strings as appropriate.
815  ResultPtr = &ResultBuf[0];   // Next byte to fill in.
816
817  Pascal = false;
818
819  for (unsigned i = 0, e = NumStringToks; i != e; ++i) {
820    const char *ThisTokBuf = &TokenBuf[0];
821    // Get the spelling of the token, which eliminates trigraphs, etc.  We know
822    // that ThisTokBuf points to a buffer that is big enough for the whole token
823    // and 'spelled' tokens can only shrink.
824    bool StringInvalid = false;
825    unsigned ThisTokLen = PP.getSpelling(StringToks[i], ThisTokBuf,
826                                         &StringInvalid);
827    if (StringInvalid) {
828      hadError = 1;
829      continue;
830    }
831
832    const char *ThisTokEnd = ThisTokBuf+ThisTokLen-1;  // Skip end quote.
833
834    // TODO: Input character set mapping support.
835
836    // Skip L marker for wide strings.
837    if (ThisTokBuf[0] == 'L')
838      ++ThisTokBuf;
839
840    assert(ThisTokBuf[0] == '"' && "Expected quote, lexer broken?");
841    ++ThisTokBuf;
842
843    // Check if this is a pascal string
844    if (pp.getLangOptions().PascalStrings && ThisTokBuf + 1 != ThisTokEnd &&
845        ThisTokBuf[0] == '\\' && ThisTokBuf[1] == 'p') {
846
847      // If the \p sequence is found in the first token, we have a pascal string
848      // Otherwise, if we already have a pascal string, ignore the first \p
849      if (i == 0) {
850        ++ThisTokBuf;
851        Pascal = true;
852      } else if (Pascal)
853        ThisTokBuf += 2;
854    }
855
856    while (ThisTokBuf != ThisTokEnd) {
857      // Is this a span of non-escape characters?
858      if (ThisTokBuf[0] != '\\') {
859        const char *InStart = ThisTokBuf;
860        do {
861          ++ThisTokBuf;
862        } while (ThisTokBuf != ThisTokEnd && ThisTokBuf[0] != '\\');
863
864        // Copy the character span over.
865        unsigned Len = ThisTokBuf-InStart;
866        if (!AnyWide) {
867          memcpy(ResultPtr, InStart, Len);
868          ResultPtr += Len;
869        } else {
870          // Note: our internal rep of wide char tokens is always little-endian.
871          for (; Len; --Len, ++InStart) {
872            *ResultPtr++ = InStart[0];
873            // Add zeros at the end.
874            for (unsigned i = 1, e = wchar_tByteWidth; i != e; ++i)
875              *ResultPtr++ = 0;
876          }
877        }
878        continue;
879      }
880      // Is this a Universal Character Name escape?
881      if (ThisTokBuf[1] == 'u' || ThisTokBuf[1] == 'U') {
882        ProcessUCNEscape(ThisTokBuf, ThisTokEnd, ResultPtr,
883                         hadError, StringToks[i].getLocation(), PP, Complain);
884        continue;
885      }
886      // Otherwise, this is a non-UCN escape character.  Process it.
887      unsigned ResultChar = ProcessCharEscape(ThisTokBuf, ThisTokEnd, hadError,
888                                              StringToks[i].getLocation(),
889                                              AnyWide, PP, Complain);
890
891      // Note: our internal rep of wide char tokens is always little-endian.
892      *ResultPtr++ = ResultChar & 0xFF;
893
894      if (AnyWide) {
895        for (unsigned i = 1, e = wchar_tByteWidth; i != e; ++i)
896          *ResultPtr++ = ResultChar >> i*8;
897      }
898    }
899  }
900
901  if (Pascal) {
902    ResultBuf[0] = ResultPtr-&ResultBuf[0]-1;
903    if (AnyWide)
904      ResultBuf[0] /= wchar_tByteWidth;
905
906    // Verify that pascal strings aren't too large.
907    if (GetStringLength() > 256 && Complain) {
908      PP.Diag(StringToks[0].getLocation(), diag::err_pascal_string_too_long)
909        << SourceRange(StringToks[0].getLocation(),
910                       StringToks[NumStringToks-1].getLocation());
911      hadError = 1;
912      return;
913    }
914  } else if (Complain) {
915    // Complain if this string literal has too many characters.
916    unsigned MaxChars = PP.getLangOptions().CPlusPlus? 65536
917                      : PP.getLangOptions().C99 ? 4095
918                      : 509;
919
920    if (GetNumStringChars() > MaxChars)
921      PP.Diag(StringToks[0].getLocation(), diag::ext_string_too_long)
922        << GetNumStringChars() << MaxChars
923        << (PP.getLangOptions().CPlusPlus? 2
924            : PP.getLangOptions().C99 ? 1
925            : 0)
926        << SourceRange(StringToks[0].getLocation(),
927                       StringToks[NumStringToks-1].getLocation());
928  }
929}
930
931
932/// getOffsetOfStringByte - This function returns the offset of the
933/// specified byte of the string data represented by Token.  This handles
934/// advancing over escape sequences in the string.
935unsigned StringLiteralParser::getOffsetOfStringByte(const Token &Tok,
936                                                    unsigned ByteNo,
937                                                    Preprocessor &PP,
938                                                    bool Complain) {
939  // Get the spelling of the token.
940  llvm::SmallString<16> SpellingBuffer;
941  SpellingBuffer.resize(Tok.getLength());
942
943  bool StringInvalid = false;
944  const char *SpellingPtr = &SpellingBuffer[0];
945  unsigned TokLen = PP.getSpelling(Tok, SpellingPtr, &StringInvalid);
946  if (StringInvalid) {
947    return 0;
948  }
949
950  assert(SpellingPtr[0] != 'L' && "Doesn't handle wide strings yet");
951
952
953  const char *SpellingStart = SpellingPtr;
954  const char *SpellingEnd = SpellingPtr+TokLen;
955
956  // Skip over the leading quote.
957  assert(SpellingPtr[0] == '"' && "Should be a string literal!");
958  ++SpellingPtr;
959
960  // Skip over bytes until we find the offset we're looking for.
961  while (ByteNo) {
962    assert(SpellingPtr < SpellingEnd && "Didn't find byte offset!");
963
964    // Step over non-escapes simply.
965    if (*SpellingPtr != '\\') {
966      ++SpellingPtr;
967      --ByteNo;
968      continue;
969    }
970
971    // Otherwise, this is an escape character.  Advance over it.
972    bool HadError = false;
973    ProcessCharEscape(SpellingPtr, SpellingEnd, HadError,
974                      Tok.getLocation(), false, PP, Complain);
975    assert(!HadError && "This method isn't valid on erroneous strings");
976    --ByteNo;
977  }
978
979  return SpellingPtr-SpellingStart;
980}
981