1 /*
  2  * Copyright (c) 1999, 2021, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.  Oracle designates this
  8  * particular file as subject to the "Classpath" exception as provided
  9  * by Oracle in the LICENSE file that accompanied this code.
 10  *
 11  * This code is distributed in the hope that it will be useful, but WITHOUT
 12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 14  * version 2 for more details (a copy is included in the LICENSE file that
 15  * accompanied this code).
 16  *
 17  * You should have received a copy of the GNU General Public License version
 18  * 2 along with this work; if not, write to the Free Software Foundation,
 19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 20  *
 21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 22  * or visit www.oracle.com if you need additional information or have any
 23  * questions.
 24  */
 25 
 26 package com.sun.tools.javac.parser;
 27 
 28 import java.util.HashMap;
 29 import java.util.Locale;
 30 import java.util.function.Predicate;
 31 import java.util.Map;
 32 
 33 import com.sun.tools.javac.api.Formattable;
 34 import com.sun.tools.javac.api.Messages;
 35 import com.sun.tools.javac.parser.Tokens.Token.Tag;
 36 import com.sun.tools.javac.util.*;
 37 
 38 /** A class that defines codes/utilities for Java source tokens
 39  *  returned from lexical analysis.
 40  *
 41  *  <p><b>This is NOT part of any supported API.
 42  *  If you write code that depends on this, you do so at your own risk.
 43  *  This code and its internal interfaces are subject to change or
 44  *  deletion without notice.</b>
 45  */
 46 public class Tokens {
 47 
 48     private final Names names;
 49 
 50     /**
 51      * Keyword array. Maps name indices to Token.
 52      */
 53     private Map<String, TokenKind> keywords = new HashMap<>();
 54 
 55     public static final Context.Key<Tokens> tokensKey = new Context.Key<>();
 56 
 57     public static Tokens instance(Context context) {
 58         Tokens instance = context.get(tokensKey);
 59         if (instance == null)
 60             instance = new Tokens(context);
 61         return instance;
 62     }
 63 
 64     protected Tokens(Context context) {
 65         context.put(tokensKey, this);
 66         names = Names.instance(context);
 67         for (TokenKind t : TokenKind.values()) {
 68             if (t.name != null) {
 69                 names.fromString(t.name);
 70                 keywords.put(t.name, t);
 71             }
 72         }
 73     }
 74 
 75     /**
 76      * Create a new token given a name; if the name corresponds to a token name,
 77      * a new token of the corresponding kind is returned; otherwise, an
 78      * identifier token is returned.
 79      */
 80     TokenKind lookupKind(Name name) {
 81         TokenKind t = keywords.get(name.toString());
 82         return (t != null) ? t : TokenKind.IDENTIFIER;
 83     }
 84 
 85     TokenKind lookupKind(String name) {
 86         TokenKind t = keywords.get(name);
 87         return (t != null) ? t : TokenKind.IDENTIFIER;
 88     }
 89 
 90     /**
 91      * This enum defines all tokens used by the javac scanner. A token is
 92      * optionally associated with a name.
 93      */
 94     public enum TokenKind implements Formattable, Predicate<TokenKind> {
 95         EOF(),
 96         ERROR(),
 97         IDENTIFIER(Tag.NAMED),
 98         ABSTRACT("abstract"),
 99         ASSERT("assert", Tag.NAMED),
100         BOOLEAN("boolean", Tag.NAMED),
101         BREAK("break"),
102         BYTE("byte", Tag.NAMED),
103         CASE("case"),
104         CATCH("catch"),
105         CHAR("char", Tag.NAMED),
106         CLASS("class"),
107         CONST("const"),
108         CONTINUE("continue"),
109         DEFAULT("default"),
110         DO("do"),
111         DOUBLE("double", Tag.NAMED),
112         ELSE("else"),
113         ENUM("enum", Tag.NAMED),
114         EXTENDS("extends"),
115         FINAL("final"),
116         FINALLY("finally"),
117         FLOAT("float", Tag.NAMED),
118         FOR("for"),
119         GOTO("goto"),
120         IF("if"),
121         IMPLEMENTS("implements"),
122         IMPORT("import"),
123         INSTANCEOF("instanceof"),
124         INT("int", Tag.NAMED),
125         INTERFACE("interface"),
126         LONG("long", Tag.NAMED),
127         NATIVE("native"),
128         NEW("new"),
129         PACKAGE("package"),
130         PRIVATE("private"),
131         PROTECTED("protected"),
132         PUBLIC("public"),
133         RETURN("return"),
134         SHORT("short", Tag.NAMED),
135         STATIC("static"),
136         STRICTFP("strictfp"),
137         SUPER("super", Tag.NAMED),
138         SWITCH("switch"),
139         SYNCHRONIZED("synchronized"),
140         THIS("this", Tag.NAMED),
141         THROW("throw"),
142         THROWS("throws"),
143         TRANSIENT("transient"),
144         TRY("try"),
145         VOID("void", Tag.NAMED),
146         VOLATILE("volatile"),
147         WHILE("while"),
148         WITHFIELD("__WithField"),
149         INTLITERAL(Tag.NUMERIC),
150         LONGLITERAL(Tag.NUMERIC),
151         FLOATLITERAL(Tag.NUMERIC),
152         DOUBLELITERAL(Tag.NUMERIC),
153         CHARLITERAL(Tag.NUMERIC),
154         STRINGLITERAL(Tag.STRING),
155         TRUE("true", Tag.NAMED),
156         FALSE("false", Tag.NAMED),
157         NULL("null", Tag.NAMED),
158         UNDERSCORE("_", Tag.NAMED),
159         ARROW("->"),
160         COLCOL("::"),
161         LPAREN("("),
162         RPAREN(")"),
163         LBRACE("{"),
164         RBRACE("}"),
165         LBRACKET("["),
166         RBRACKET("]"),
167         SEMI(";"),
168         COMMA(","),
169         DOT("."),
170         ELLIPSIS("..."),
171         EQ("="),
172         GT(">"),
173         LT("<"),
174         BANG("!"),
175         TILDE("~"),
176         QUES("?"),
177         COLON(":"),
178         EQEQ("=="),
179         LTEQ("<="),
180         GTEQ(">="),
181         BANGEQ("!="),
182         AMPAMP("&&"),
183         BARBAR("||"),
184         PLUSPLUS("++"),
185         SUBSUB("--"),
186         PLUS("+"),
187         SUB("-"),
188         STAR("*"),
189         SLASH("/"),
190         AMP("&"),
191         BAR("|"),
192         CARET("^"),
193         PERCENT("%"),
194         LTLT("<<"),
195         GTGT(">>"),
196         GTGTGT(">>>"),
197         PLUSEQ("+="),
198         SUBEQ("-="),
199         STAREQ("*="),
200         SLASHEQ("/="),
201         AMPEQ("&="),
202         BAREQ("|="),
203         CARETEQ("^="),
204         PERCENTEQ("%="),
205         LTLTEQ("<<="),
206         GTGTEQ(">>="),
207         GTGTGTEQ(">>>="),
208         MONKEYS_AT("@"),
209         CUSTOM;
210 
211         public final String name;
212         public final Tag tag;
213 
214         TokenKind() {
215             this(null, Tag.DEFAULT);
216         }
217 
218         TokenKind(String name) {
219             this(name, Tag.DEFAULT);
220         }
221 
222         TokenKind(Tag tag) {
223             this(null, tag);
224         }
225 
226         TokenKind(String name, Tag tag) {
227             this.name = name;
228             this.tag = tag;
229         }
230 
231         public String toString() {
232             switch (this) {
233             case IDENTIFIER:
234                 return "token.identifier";
235             case CHARLITERAL:
236                 return "token.character";
237             case STRINGLITERAL:
238                 return "token.string";
239             case INTLITERAL:
240                 return "token.integer";
241             case LONGLITERAL:
242                 return "token.long-integer";
243             case FLOATLITERAL:
244                 return "token.float";
245             case DOUBLELITERAL:
246                 return "token.double";
247             case ERROR:
248                 return "token.bad-symbol";
249             case EOF:
250                 return "token.end-of-input";
251             case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:
252             case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:
253                 return "'" + name + "'";
254             default:
255                 return name;
256             }
257         }
258 
259         public String getKind() {
260             return "Token";
261         }
262 
263         public String toString(Locale locale, Messages messages) {
264             return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());
265         }
266 
267         @Override
268         public boolean test(TokenKind that) {
269             return this == that;
270         }
271     }
272 
273     public interface Comment {
274 
275         enum CommentStyle {
276             LINE,
277             BLOCK,
278             JAVADOC,
279         }
280 
281         String getText();
282         int getSourcePos(int index);
283         CommentStyle getStyle();
284         boolean isDeprecated();
285     }
286 
287     /**
288      * This is the class representing a javac token. Each token has several fields
289      * that are set by the javac lexer (i.e. start/end position, string value, etc).
290      */
291     public static class Token {
292 
293         /** tags constants **/
294         enum Tag {
295             DEFAULT,
296             NAMED,
297             STRING,
298             NUMERIC;
299         }
300 
301         /** The token kind */
302         public final TokenKind kind;
303 
304         /** The start position of this token */
305         public final int pos;
306 
307         /** The end position of this token */
308         public final int endPos;
309 
310         /** Comment reader associated with this token */
311         public final List<Comment> comments;
312 
313         Token(TokenKind kind, int pos, int endPos, List<Comment> comments) {
314             this.kind = kind;
315             this.pos = pos;
316             this.endPos = endPos;
317             this.comments = comments;
318             checkKind();
319         }
320 
321         Token[] split(Tokens tokens) {
322             if (kind.name.length() < 2 || kind.tag != Tag.DEFAULT) {
323                 throw new AssertionError("Cant split" + kind);
324             }
325 
326             TokenKind t1 = tokens.lookupKind(kind.name.substring(0, 1));
327             TokenKind t2 = tokens.lookupKind(kind.name.substring(1));
328 
329             if (t1 == null || t2 == null) {
330                 throw new AssertionError("Cant split - bad subtokens");
331             }
332             return new Token[] {
333                 new Token(t1, pos, pos + t1.name.length(), comments),
334                 new Token(t2, pos + t1.name.length(), endPos, null)
335             };
336         }
337 
338         protected void checkKind() {
339             if (kind.tag != Tag.DEFAULT) {
340                 throw new AssertionError("Bad token kind - expected " + Tag.DEFAULT);
341             }
342         }
343 
344         public Name name() {
345             throw new UnsupportedOperationException();
346         }
347 
348         public String stringVal() {
349             throw new UnsupportedOperationException();
350         }
351 
352         public int radix() {
353             throw new UnsupportedOperationException();
354         }
355 
356         /**
357          * Preserve classic semantics - if multiple javadocs are found on the token
358          * the last one is returned
359          */
360         public Comment comment(Comment.CommentStyle style) {
361             List<Comment> comments = getComments(Comment.CommentStyle.JAVADOC);
362             return comments.isEmpty() ?
363                     null :
364                     comments.head;
365         }
366 
367         /**
368          * Preserve classic semantics - deprecated should be set if at least one
369          * javadoc comment attached to this token contains the '@deprecated' string
370          */
371         public boolean deprecatedFlag() {
372             for (Comment c : getComments(Comment.CommentStyle.JAVADOC)) {
373                 if (c.isDeprecated()) {
374                     return true;
375                 }
376             }
377             return false;
378         }
379 
380         private List<Comment> getComments(Comment.CommentStyle style) {
381             if (comments == null) {
382                 return List.nil();
383             } else {
384                 ListBuffer<Comment> buf = new ListBuffer<>();
385                 for (Comment c : comments) {
386                     if (c.getStyle() == style) {
387                         buf.add(c);
388                     }
389                 }
390                 return buf.toList();
391             }
392         }
393     }
394 
395     static final class NamedToken extends Token {
396         /** The name of this token */
397         public final Name name;
398 
399         public NamedToken(TokenKind kind, int pos, int endPos, Name name, List<Comment> comments) {
400             super(kind, pos, endPos, comments);
401             this.name = name;
402         }
403 
404         protected void checkKind() {
405             if (kind.tag != Tag.NAMED) {
406                 throw new AssertionError("Bad token kind - expected " + Tag.NAMED);
407             }
408         }
409 
410         @Override
411         public Name name() {
412             return name;
413         }
414     }
415 
416     static class StringToken extends Token {
417         /** The string value of this token */
418         public final String stringVal;
419 
420         public StringToken(TokenKind kind, int pos, int endPos, String stringVal, List<Comment> comments) {
421             super(kind, pos, endPos, comments);
422             this.stringVal = stringVal;
423         }
424 
425         protected void checkKind() {
426             if (kind.tag != Tag.STRING) {
427                 throw new AssertionError("Bad token kind - expected " + Tag.STRING);
428             }
429         }
430 
431         @Override
432         public String stringVal() {
433             return stringVal;
434         }
435     }
436 
437     static final class NumericToken extends StringToken {
438         /** The 'radix' value of this token */
439         public final int radix;
440 
441         public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, List<Comment> comments) {
442             super(kind, pos, endPos, stringVal, comments);
443             this.radix = radix;
444         }
445 
446         protected void checkKind() {
447             if (kind.tag != Tag.NUMERIC) {
448                 throw new AssertionError("Bad token kind - expected " + Tag.NUMERIC);
449             }
450         }
451 
452         @Override
453         public int radix() {
454             return radix;
455         }
456     }
457 
458     public static final Token DUMMY =
459                 new Token(TokenKind.ERROR, 0, 0, null);
460 }