1/*
2 * Copyright (C) 2007-2010 Júlio Vilmar Gesser.
3 * Copyright (C) 2011, 2013-2016 The JavaParser Team.
4 *
5 * This file is part of JavaParser.
6 *
7 * JavaParser can be used either under the terms of
8 * a) the GNU Lesser General Public License as published by
9 *     the Free Software Foundation, either version 3 of the License, or
10 *     (at your option) any later version.
11 * b) the terms of the Apache License
12 *
13 * You should have received a copy of both licenses in LICENCE.LGPL and
14 * LICENCE.APACHE. Please refer to those files for details.
15 *
16 * JavaParser is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
19 * GNU Lesser General Public License for more details.
20 */
21package com.github.javaparser;
22
23import java.util.List;
24import java.util.Optional;
25import static com.github.javaparser.utils.CodeGenerationUtils.f;
26import static com.github.javaparser.utils.Utils.EOL;
27import static com.github.javaparser.utils.Utils.assertNotNull;
28import javax.annotation.Generated;
29
30/**
31 * A token from a parsed source file.
32 * (Awkwardly named "Java"Token since JavaCC already generates an internal class Token.)
33 * It is a node in a double linked list called token list.
34 */
35public class JavaToken {
36
37    public static final JavaToken INVALID = new JavaToken();
38
39    private Range range;
40
41    private int kind;
42
43    private String text;
44
45    private JavaToken previousToken = null;
46
47    private JavaToken nextToken = null;
48
49    private JavaToken() {
50        this(null, 0, "INVALID", null, null);
51    }
52
53    public JavaToken(int kind, String text) {
54        this(null, kind, text, null, null);
55    }
56
57    JavaToken(Token token, List<JavaToken> tokens) {
58        // You could be puzzled by the following lines
59        //
60        // The reason why these lines are necessary is the fact that Java is ambiguous. There are cases where the
61        // sequence of characters ">>>" and ">>" should be recognized as the single tokens ">>>" and ">>". In other
62        // cases however we want to split those characters in single GT tokens (">").
63        //
64        // For example, in expressions ">>" and ">>>" are valid, while when defining types we could have this:
65        //
66        // List<List<Set<String>>>>
67        //
68        // You can see that the sequence ">>>>" should be interpreted as four consecutive ">" tokens closing a type
69        // parameter list.
70        //
71        // The JavaCC handle this case by first recognizing always the longest token, and then depending on the context
72        // putting back the unused chars in the stream. However in those cases the token provided is invalid: it has an
73        // image corresponding to the text originally recognized, without considering that after some characters could
74        // have been put back into the stream.
75        //
76        // So in the case of:
77        //
78        // List<List<Set<String>>>>
79        // ___   -> recognized as ">>>", then ">>" put back in the stream but Token(type=GT, image=">>>") passed to this class
80        // ___  -> recognized as ">>>", then ">>" put back in the stream but Token(type=GT, image=">>>") passed to this class
81        // __  -> recognized as ">>", then ">" put back in the stream but Token(type=GT, image=">>") passed to this class
82        // _  -> Token(type=GT, image=">") good!
83        //
84        // So given the image could be wrong but the type is correct, we look at the type of the token and we fix
85        // the image. Everybody is happy and we can keep this horrible thing as our little secret.
86        Range range = Range.range(token.beginLine, token.beginColumn, token.endLine, token.endColumn);
87        String text = token.image;
88        if (token.kind == GeneratedJavaParserConstants.GT) {
89            range = Range.range(token.beginLine, token.beginColumn, token.endLine, token.beginColumn);
90            text = ">";
91        } else if (token.kind == GeneratedJavaParserConstants.RSIGNEDSHIFT) {
92            range = Range.range(token.beginLine, token.beginColumn, token.endLine, token.beginColumn + 1);
93            text = ">>";
94        }
95        this.range = range;
96        this.kind = token.kind;
97        this.text = text;
98        if (!tokens.isEmpty()) {
99            final JavaToken previousToken = tokens.get(tokens.size() - 1);
100            this.previousToken = previousToken;
101            previousToken.nextToken = this;
102        } else {
103            previousToken = null;
104        }
105    }
106
107    /**
108     * Create a token of a certain kind.
109     */
110    public JavaToken(int kind) {
111        String content = GeneratedJavaParserConstants.tokenImage[kind];
112        if (content.startsWith("\"")) {
113            content = content.substring(1, content.length() - 1);
114        }
115        if (TokenTypes.isEndOfLineToken(kind)) {
116            content = EOL;
117        } else if (TokenTypes.isWhitespace(kind)) {
118            content = " ";
119        }
120        this.kind = kind;
121        this.text = content;
122    }
123
124    public JavaToken(Range range, int kind, String text, JavaToken previousToken, JavaToken nextToken) {
125        assertNotNull(text);
126        this.range = range;
127        this.kind = kind;
128        this.text = text;
129        this.previousToken = previousToken;
130        this.nextToken = nextToken;
131    }
132
133    public Optional<Range> getRange() {
134        return Optional.ofNullable(range);
135    }
136
137    public int getKind() {
138        return kind;
139    }
140
141    void setKind(int kind) {
142        this.kind = kind;
143    }
144
145    public String getText() {
146        return text;
147    }
148
149    public Optional<JavaToken> getNextToken() {
150        return Optional.ofNullable(nextToken);
151    }
152
153    public Optional<JavaToken> getPreviousToken() {
154        return Optional.ofNullable(previousToken);
155    }
156
157    public void setRange(Range range) {
158        this.range = range;
159    }
160
161    public void setText(String text) {
162        this.text = text;
163    }
164
165    public String asString() {
166        return text;
167    }
168
169    /**
170     * @return the token range that goes from the beginning to the end of the token list this token is a part of.
171     */
172    public TokenRange toTokenRange() {
173        return new TokenRange(findFirstToken(), findLastToken());
174    }
175
176    @Override
177    public String toString() {
178        String text = getText().replace("\n", "\\n").replace("\r", "\\r").replace("\r\n", "\\r\\n").replace("\t", "\\t");
179        return f("\"%s\"   <%s>   %s", text, getKind(), getRange().map(Range::toString).orElse("(?)-(?)"));
180    }
181
182    /**
183     * Used by the parser while constructing nodes. No tokens should be invalid when the parser is done.
184     */
185    public boolean valid() {
186        return !invalid();
187    }
188
189    /**
190     * Used by the parser while constructing nodes. No tokens should be invalid when the parser is done.
191     */
192    public boolean invalid() {
193        return this == INVALID;
194    }
195
196    public enum Category {
197
198        WHITESPACE_NO_EOL,
199        EOL,
200        COMMENT,
201        IDENTIFIER,
202        KEYWORD,
203        LITERAL,
204        SEPARATOR,
205        OPERATOR;
206
207        public boolean isWhitespaceOrComment() {
208            return isWhitespace() || this == COMMENT;
209        }
210
211        public boolean isWhitespace() {
212            return this == WHITESPACE_NO_EOL || this == EOL;
213        }
214
215        public boolean isEndOfLine() {
216            return this == EOL;
217        }
218
219        public boolean isComment() {
220            return this == COMMENT;
221        }
222
223        public boolean isWhitespaceButNotEndOfLine() {
224            return this == WHITESPACE_NO_EOL;
225        }
226
227        public boolean isIdentifier() {
228            return this == IDENTIFIER;
229        }
230
231        public boolean isKeyword() {
232            return this == KEYWORD;
233        }
234
235        public boolean isLiteral() {
236            return this == LITERAL;
237        }
238
239        public boolean isSeparator() {
240            return this == SEPARATOR;
241        }
242
243        public boolean isOperator() {
244            return this == OPERATOR;
245        }
246    }
247
248    @Generated("com.github.javaparser.generator.core.other.TokenKindGenerator")
249    public enum Kind {
250
251        EOF(0),
252        SPACE(1),
253        WINDOWS_EOL(2),
254        UNIX_EOL(3),
255        OLD_MAC_EOL(4),
256        SINGLE_LINE_COMMENT(5),
257        ENTER_JAVADOC_COMMENT(6),
258        ENTER_MULTILINE_COMMENT(7),
259        JAVADOC_COMMENT(8),
260        MULTI_LINE_COMMENT(9),
261        COMMENT_CONTENT(10),
262        ABSTRACT(11),
263        ASSERT(12),
264        BOOLEAN(13),
265        BREAK(14),
266        BYTE(15),
267        CASE(16),
268        CATCH(17),
269        CHAR(18),
270        CLASS(19),
271        CONST(20),
272        CONTINUE(21),
273        _DEFAULT(22),
274        DO(23),
275        DOUBLE(24),
276        ELSE(25),
277        ENUM(26),
278        EXTENDS(27),
279        FALSE(28),
280        FINAL(29),
281        FINALLY(30),
282        FLOAT(31),
283        FOR(32),
284        GOTO(33),
285        IF(34),
286        IMPLEMENTS(35),
287        IMPORT(36),
288        INSTANCEOF(37),
289        INT(38),
290        INTERFACE(39),
291        LONG(40),
292        NATIVE(41),
293        NEW(42),
294        NULL(43),
295        PACKAGE(44),
296        PRIVATE(45),
297        PROTECTED(46),
298        PUBLIC(47),
299        RETURN(48),
300        SHORT(49),
301        STATIC(50),
302        STRICTFP(51),
303        SUPER(52),
304        SWITCH(53),
305        SYNCHRONIZED(54),
306        THIS(55),
307        THROW(56),
308        THROWS(57),
309        TRANSIENT(58),
310        TRUE(59),
311        TRY(60),
312        VOID(61),
313        VOLATILE(62),
314        WHILE(63),
315        REQUIRES(64),
316        TO(65),
317        WITH(66),
318        OPEN(67),
319        OPENS(68),
320        USES(69),
321        MODULE(70),
322        EXPORTS(71),
323        PROVIDES(72),
324        TRANSITIVE(73),
325        LONG_LITERAL(74),
326        INTEGER_LITERAL(75),
327        DECIMAL_LITERAL(76),
328        HEX_LITERAL(77),
329        OCTAL_LITERAL(78),
330        BINARY_LITERAL(79),
331        FLOATING_POINT_LITERAL(80),
332        DECIMAL_FLOATING_POINT_LITERAL(81),
333        DECIMAL_EXPONENT(82),
334        HEXADECIMAL_FLOATING_POINT_LITERAL(83),
335        HEXADECIMAL_EXPONENT(84),
336        HEX_DIGITS(85),
337        UNICODE_ESCAPE(86),
338        CHARACTER_LITERAL(87),
339        STRING_LITERAL(88),
340        IDENTIFIER(89),
341        LETTER(90),
342        PART_LETTER(91),
343        LPAREN(92),
344        RPAREN(93),
345        LBRACE(94),
346        RBRACE(95),
347        LBRACKET(96),
348        RBRACKET(97),
349        SEMICOLON(98),
350        COMMA(99),
351        DOT(100),
352        AT(101),
353        ASSIGN(102),
354        LT(103),
355        BANG(104),
356        TILDE(105),
357        HOOK(106),
358        COLON(107),
359        EQ(108),
360        LE(109),
361        GE(110),
362        NE(111),
363        SC_OR(112),
364        SC_AND(113),
365        INCR(114),
366        DECR(115),
367        PLUS(116),
368        MINUS(117),
369        STAR(118),
370        SLASH(119),
371        BIT_AND(120),
372        BIT_OR(121),
373        XOR(122),
374        REM(123),
375        LSHIFT(124),
376        PLUSASSIGN(125),
377        MINUSASSIGN(126),
378        STARASSIGN(127),
379        SLASHASSIGN(128),
380        ANDASSIGN(129),
381        ORASSIGN(130),
382        XORASSIGN(131),
383        REMASSIGN(132),
384        LSHIFTASSIGN(133),
385        RSIGNEDSHIFTASSIGN(134),
386        RUNSIGNEDSHIFTASSIGN(135),
387        ELLIPSIS(136),
388        ARROW(137),
389        DOUBLECOLON(138),
390        RUNSIGNEDSHIFT(139),
391        RSIGNEDSHIFT(140),
392        GT(141),
393        CTRL_Z(142);
394
395        private final int kind;
396
397        Kind(int kind) {
398            this.kind = kind;
399        }
400
401        public static Kind valueOf(int kind) {
402            switch(kind) {
403                case 142:
404                    return CTRL_Z;
405                case 141:
406                    return GT;
407                case 140:
408                    return RSIGNEDSHIFT;
409                case 139:
410                    return RUNSIGNEDSHIFT;
411                case 138:
412                    return DOUBLECOLON;
413                case 137:
414                    return ARROW;
415                case 136:
416                    return ELLIPSIS;
417                case 135:
418                    return RUNSIGNEDSHIFTASSIGN;
419                case 134:
420                    return RSIGNEDSHIFTASSIGN;
421                case 133:
422                    return LSHIFTASSIGN;
423                case 132:
424                    return REMASSIGN;
425                case 131:
426                    return XORASSIGN;
427                case 130:
428                    return ORASSIGN;
429                case 129:
430                    return ANDASSIGN;
431                case 128:
432                    return SLASHASSIGN;
433                case 127:
434                    return STARASSIGN;
435                case 126:
436                    return MINUSASSIGN;
437                case 125:
438                    return PLUSASSIGN;
439                case 124:
440                    return LSHIFT;
441                case 123:
442                    return REM;
443                case 122:
444                    return XOR;
445                case 121:
446                    return BIT_OR;
447                case 120:
448                    return BIT_AND;
449                case 119:
450                    return SLASH;
451                case 118:
452                    return STAR;
453                case 117:
454                    return MINUS;
455                case 116:
456                    return PLUS;
457                case 115:
458                    return DECR;
459                case 114:
460                    return INCR;
461                case 113:
462                    return SC_AND;
463                case 112:
464                    return SC_OR;
465                case 111:
466                    return NE;
467                case 110:
468                    return GE;
469                case 109:
470                    return LE;
471                case 108:
472                    return EQ;
473                case 107:
474                    return COLON;
475                case 106:
476                    return HOOK;
477                case 105:
478                    return TILDE;
479                case 104:
480                    return BANG;
481                case 103:
482                    return LT;
483                case 102:
484                    return ASSIGN;
485                case 101:
486                    return AT;
487                case 100:
488                    return DOT;
489                case 99:
490                    return COMMA;
491                case 98:
492                    return SEMICOLON;
493                case 97:
494                    return RBRACKET;
495                case 96:
496                    return LBRACKET;
497                case 95:
498                    return RBRACE;
499                case 94:
500                    return LBRACE;
501                case 93:
502                    return RPAREN;
503                case 92:
504                    return LPAREN;
505                case 91:
506                    return PART_LETTER;
507                case 90:
508                    return LETTER;
509                case 89:
510                    return IDENTIFIER;
511                case 88:
512                    return STRING_LITERAL;
513                case 87:
514                    return CHARACTER_LITERAL;
515                case 86:
516                    return UNICODE_ESCAPE;
517                case 85:
518                    return HEX_DIGITS;
519                case 84:
520                    return HEXADECIMAL_EXPONENT;
521                case 83:
522                    return HEXADECIMAL_FLOATING_POINT_LITERAL;
523                case 82:
524                    return DECIMAL_EXPONENT;
525                case 81:
526                    return DECIMAL_FLOATING_POINT_LITERAL;
527                case 80:
528                    return FLOATING_POINT_LITERAL;
529                case 79:
530                    return BINARY_LITERAL;
531                case 78:
532                    return OCTAL_LITERAL;
533                case 77:
534                    return HEX_LITERAL;
535                case 76:
536                    return DECIMAL_LITERAL;
537                case 75:
538                    return INTEGER_LITERAL;
539                case 74:
540                    return LONG_LITERAL;
541                case 73:
542                    return TRANSITIVE;
543                case 72:
544                    return PROVIDES;
545                case 71:
546                    return EXPORTS;
547                case 70:
548                    return MODULE;
549                case 69:
550                    return USES;
551                case 68:
552                    return OPENS;
553                case 67:
554                    return OPEN;
555                case 66:
556                    return WITH;
557                case 65:
558                    return TO;
559                case 64:
560                    return REQUIRES;
561                case 63:
562                    return WHILE;
563                case 62:
564                    return VOLATILE;
565                case 61:
566                    return VOID;
567                case 60:
568                    return TRY;
569                case 59:
570                    return TRUE;
571                case 58:
572                    return TRANSIENT;
573                case 57:
574                    return THROWS;
575                case 56:
576                    return THROW;
577                case 55:
578                    return THIS;
579                case 54:
580                    return SYNCHRONIZED;
581                case 53:
582                    return SWITCH;
583                case 52:
584                    return SUPER;
585                case 51:
586                    return STRICTFP;
587                case 50:
588                    return STATIC;
589                case 49:
590                    return SHORT;
591                case 48:
592                    return RETURN;
593                case 47:
594                    return PUBLIC;
595                case 46:
596                    return PROTECTED;
597                case 45:
598                    return PRIVATE;
599                case 44:
600                    return PACKAGE;
601                case 43:
602                    return NULL;
603                case 42:
604                    return NEW;
605                case 41:
606                    return NATIVE;
607                case 40:
608                    return LONG;
609                case 39:
610                    return INTERFACE;
611                case 38:
612                    return INT;
613                case 37:
614                    return INSTANCEOF;
615                case 36:
616                    return IMPORT;
617                case 35:
618                    return IMPLEMENTS;
619                case 34:
620                    return IF;
621                case 33:
622                    return GOTO;
623                case 32:
624                    return FOR;
625                case 31:
626                    return FLOAT;
627                case 30:
628                    return FINALLY;
629                case 29:
630                    return FINAL;
631                case 28:
632                    return FALSE;
633                case 27:
634                    return EXTENDS;
635                case 26:
636                    return ENUM;
637                case 25:
638                    return ELSE;
639                case 24:
640                    return DOUBLE;
641                case 23:
642                    return DO;
643                case 22:
644                    return _DEFAULT;
645                case 21:
646                    return CONTINUE;
647                case 20:
648                    return CONST;
649                case 19:
650                    return CLASS;
651                case 18:
652                    return CHAR;
653                case 17:
654                    return CATCH;
655                case 16:
656                    return CASE;
657                case 15:
658                    return BYTE;
659                case 14:
660                    return BREAK;
661                case 13:
662                    return BOOLEAN;
663                case 12:
664                    return ASSERT;
665                case 11:
666                    return ABSTRACT;
667                case 10:
668                    return COMMENT_CONTENT;
669                case 9:
670                    return MULTI_LINE_COMMENT;
671                case 8:
672                    return JAVADOC_COMMENT;
673                case 7:
674                    return ENTER_MULTILINE_COMMENT;
675                case 6:
676                    return ENTER_JAVADOC_COMMENT;
677                case 5:
678                    return SINGLE_LINE_COMMENT;
679                case 4:
680                    return OLD_MAC_EOL;
681                case 3:
682                    return UNIX_EOL;
683                case 2:
684                    return WINDOWS_EOL;
685                case 1:
686                    return SPACE;
687                case 0:
688                    return EOF;
689                default:
690                    throw new IllegalArgumentException(f("Token kind %i is unknown.", kind));
691            }
692        }
693
694        public int getKind() {
695            return kind;
696        }
697    }
698
699    public JavaToken.Category getCategory() {
700        return TokenTypes.getCategory(kind);
701    }
702
703    /**
704     * Inserts newToken into the token list just before this token.
705     */
706    public void insert(JavaToken newToken) {
707        assertNotNull(newToken);
708        getPreviousToken().ifPresent(p -> {
709            p.nextToken = newToken;
710            newToken.previousToken = p;
711        });
712        previousToken = newToken;
713        newToken.nextToken = this;
714    }
715
716    /**
717     * Inserts newToken into the token list just after this token.
718     */
719    public void insertAfter(JavaToken newToken) {
720        assertNotNull(newToken);
721        getNextToken().ifPresent(n -> {
722            n.previousToken = newToken;
723            newToken.nextToken = n;
724        });
725        nextToken = newToken;
726        newToken.previousToken = this;
727    }
728
729    /**
730     * Links the tokens around the current token together, making the current token disappear from the list.
731     */
732    public void deleteToken() {
733        final Optional<JavaToken> nextToken = getNextToken();
734        final Optional<JavaToken> previousToken = getPreviousToken();
735        previousToken.ifPresent(p -> p.nextToken = nextToken.orElse(null));
736        nextToken.ifPresent(n -> n.previousToken = previousToken.orElse(null));
737    }
738
739    /**
740     * Replaces the current token with newToken.
741     */
742    public void replaceToken(JavaToken newToken) {
743        assertNotNull(newToken);
744        getPreviousToken().ifPresent(p -> {
745            p.nextToken = newToken;
746            newToken.previousToken = p;
747        });
748        getNextToken().ifPresent(n -> {
749            n.previousToken = newToken;
750            newToken.nextToken = n;
751        });
752    }
753
754    /**
755     * @return the last token in the token list.
756     */
757    public JavaToken findLastToken() {
758        JavaToken current = this;
759        while (current.getNextToken().isPresent()) {
760            current = current.getNextToken().get();
761        }
762        return current;
763    }
764
765    /**
766     * @return the first token in the token list.
767     */
768    public JavaToken findFirstToken() {
769        JavaToken current = this;
770        while (current.getPreviousToken().isPresent()) {
771            current = current.getPreviousToken().get();
772        }
773        return current;
774    }
775
776    @Override
777    public int hashCode() {
778        int result = kind;
779        result = 31 * result + text.hashCode();
780        return result;
781    }
782
783    @Override
784    public boolean equals(Object o) {
785        if (this == o)
786            return true;
787        if (o == null || getClass() != o.getClass())
788            return false;
789        JavaToken javaToken = (JavaToken) o;
790        if (kind != javaToken.kind)
791            return false;
792        if (!text.equals(javaToken.text))
793            return false;
794        return true;
795    }
796}
797