1 /*
2 * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26 package java.lang.reflect.code.parser.impl;
27
28 import java.util.HashMap;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.function.Predicate;
32
33 /**
34 * A class that defines codes/utilities for Java source tokens
35 * returned from lexical analysis.
36 *
37 * <p><b>This is NOT part of any supported API.
38 * If you write code that depends on this, you do so at your own risk.
39 * This code and its internal interfaces are subject to change or
40 * deletion without notice.</b>
41 */
42 public final class Tokens {
43
44 /**
45 * Keyword array. Maps name indices to Token.
46 */
47 private final Map<String, TokenKind> keywords = new HashMap<>();
48
49 Tokens() {
50 for (TokenKind t : TokenKind.values()) {
51 if (t.name != null) {
52 keywords.put(t.name, t);
53 }
54 }
55 }
56
57 TokenKind lookupKind(String name, TokenKind identifier) {
58 TokenKind t = keywords.get(name);
59 return (t != null) ? t : identifier;
60 }
61
62 /**
63 * This enum defines all tokens used by the javac scanner. A token is
64 * optionally associated with a name.
65 */
66 public enum TokenKind implements Predicate<TokenKind> {
67 EOF(),
68 ERROR(),
69 IDENTIFIER(Token.Tag.NAMED),
70 VALUE_IDENTIFIER(Token.Tag.NAMED),
71 EXTENDS("extends", Token.Tag.NAMED),
72 SUPER("super", Token.Tag.NAMED),
73 INTLITERAL(Token.Tag.NUMERIC),
74 LONGLITERAL(Token.Tag.NUMERIC),
75 FLOATLITERAL(Token.Tag.NUMERIC),
76 DOUBLELITERAL(Token.Tag.NUMERIC),
77 CHARLITERAL(Token.Tag.NUMERIC),
78 STRINGLITERAL(Token.Tag.STRING),
79 TRUE("true", Token.Tag.NAMED),
80 FALSE("false", Token.Tag.NAMED),
81 NULL("null", Token.Tag.NAMED),
82 UNDERSCORE("_", Token.Tag.NAMED),
83 ARROW("->"),
84 LPAREN("("),
85 RPAREN(")"),
86 LBRACE("{"),
87 RBRACE("}"),
88 LBRACKET("["),
89 RBRACKET("]"),
90 COMMA(","),
91 DOT("."),
92 EQ("="),
93 GT(">"),
94 LT("<"),
95 QUES("?"),
96 COLON(":"),
97 COLCOL("::"),
98 SEMI(";"),
99 PLUS("+"),
100 SUB("-"),
101 AMP("&"),
102 CARET("^"),
103 MONKEYS_AT("@"),
104 HASH("#"),
105 CUSTOM;
106
107 public final String name;
108 public final Token.Tag tag;
109
110 TokenKind() {
111 this(null, Token.Tag.DEFAULT);
112 }
113
114 TokenKind(String name) {
115 this(name, Token.Tag.DEFAULT);
116 }
117
118 TokenKind(Token.Tag tag) {
119 this(null, tag);
120 }
121
122 TokenKind(String name, Token.Tag tag) {
123 this.name = name;
124 this.tag = tag;
125 }
126
127 public String toString() {
128 return switch (this) {
129 case IDENTIFIER -> "token.identifier";
130 case VALUE_IDENTIFIER -> "token.value-identifier";
131 case CHARLITERAL -> "token.character";
132 case STRINGLITERAL -> "token.string";
133 case INTLITERAL -> "token.integer";
134 case LONGLITERAL -> "token.long-integer";
135 case FLOATLITERAL -> "token.float";
136 case DOUBLELITERAL -> "token.double";
137 case ERROR -> "token.bad-symbol";
138 case EOF -> "token.end-of-input";
139 case DOT, COMMA, LPAREN, RPAREN, LBRACKET, RBRACKET, LBRACE, RBRACE -> "'" + name + "'";
140 default -> name;
141 };
142 }
143
144 @Override
145 public boolean test(TokenKind that) {
146 return this == that;
147 }
148 }
149
150 public interface Comment {
151
152 enum CommentStyle {
153 LINE,
154 BLOCK,
155 }
156
157 String text();
158
159 CommentStyle style();
160 }
161
162 /**
163 * This is the class representing a javac token. Each token has several fields
164 * that are set by the javac lexer (i.e. start/end position, string value, etc).
165 */
166 public static class Token {
167
168 /**
169 * tags constants
170 **/
171 public enum Tag {
172 DEFAULT,
173 NAMED,
174 STRING,
175 NUMERIC
176 }
177
178 /**
179 * The token kind
180 */
181 public final TokenKind kind;
182
183 /**
184 * The start position of this token
185 */
186 public final int pos;
187
188 /**
189 * The end position of this token
190 */
191 public final int endPos;
192
193 /**
194 * Comment reader associated with this token
195 */
196 public final List<Comment> comments;
197
198 Token(TokenKind kind, int pos, int endPos, List<Comment> comments) {
199 this.kind = kind;
200 this.pos = pos;
201 this.endPos = endPos;
202 this.comments = comments == null ? null : List.copyOf(comments);
203 checkKind();
204 }
205
206 void checkKind() {
207 if (kind.tag != Tag.DEFAULT) {
208 throw new AssertionError("Bad token kind - expected " + Tag.DEFAULT);
209 }
210 }
211
212 public String name() {
213 throw new UnsupportedOperationException();
214 }
215
216 public String stringVal() {
217 throw new UnsupportedOperationException();
218 }
219
220 public int radix() {
221 throw new UnsupportedOperationException();
222 }
223 }
224
225 static final class NamedToken extends Token {
226 /**
227 * The name of this token
228 */
229 public final String name;
230
231 NamedToken(TokenKind kind, int pos, int endPos, String name, List<Comment> comments) {
232 super(kind, pos, endPos, comments);
233 this.name = name;
234 }
235
236 void checkKind() {
237 if (kind.tag != Tag.NAMED) {
238 throw new AssertionError("Bad token kind - expected " + Tag.NAMED);
239 }
240 }
241
242 @Override
243 public String name() {
244 return name;
245 }
246 }
247
248 static class StringToken extends Token {
249 /**
250 * The string value of this token
251 */
252 public final String stringVal;
253
254 StringToken(TokenKind kind, int pos, int endPos, String stringVal, List<Comment> comments) {
255 super(kind, pos, endPos, comments);
256 this.stringVal = stringVal;
257 }
258
259 void checkKind() {
260 if (kind.tag != Tag.STRING) {
261 throw new AssertionError("Bad token kind - expected " + Tag.STRING);
262 }
263 }
264
265 @Override
266 public String stringVal() {
267 return stringVal;
268 }
269 }
270
271 static final class NumericToken extends StringToken {
272 /**
273 * The 'radix' value of this token
274 */
275 public final int radix;
276
277 NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, List<Comment> comments) {
278 super(kind, pos, endPos, stringVal, comments);
279 this.radix = radix;
280 }
281
282 void checkKind() {
283 if (kind.tag != Tag.NUMERIC) {
284 throw new AssertionError("Bad token kind - expected " + Tag.NUMERIC);
285 }
286 }
287
288 @Override
289 public int radix() {
290 return radix;
291 }
292 }
293
294 public static final Token DUMMY =
295 new Token(TokenKind.ERROR, 0, 0, null);
296 }