1 /*
2 * Copyright (c) 2024, Oracle and/or its affiliates. All rights reserved.
3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
4 *
5 * This code is free software; you can redistribute it and/or modify it
6 * under the terms of the GNU General Public License version 2 only, as
7 * published by the Free Software Foundation. Oracle designates this
8 * particular file as subject to the "Classpath" exception as provided
9 * by Oracle in the LICENSE file that accompanied this code.
10 *
11 * This code is distributed in the hope that it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 * version 2 for more details (a copy is included in the LICENSE file that
15 * accompanied this code).
16 *
17 * You should have received a copy of the GNU General Public License version
18 * 2 along with this work; if not, write to the Free Software Foundation,
19 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 *
21 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 * or visit www.oracle.com if you need additional information or have any
23 * questions.
24 */
25
26 package jdk.incubator.code.parser.impl;
27
28 import java.util.HashMap;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.function.Predicate;
32
33 /**
34 * A class that defines codes/utilities for Java source tokens
35 * returned from lexical analysis.
36 *
37 * <p><b>This is NOT part of any supported API.
38 * If you write code that depends on this, you do so at your own risk.
39 * This code and its internal interfaces are subject to change or
40 * deletion without notice.</b>
41 */
42 public final class Tokens {
43
44 /**
45 * Keyword array. Maps name indices to Token.
46 */
47 private final Map<String, TokenKind> keywords = new HashMap<>();
48
49 Tokens() {
50 for (TokenKind t : TokenKind.values()) {
51 if (t.name != null) {
52 keywords.put(t.name, t);
53 }
54 }
55 }
56
57 TokenKind lookupKind(String name, TokenKind identifier) {
58 TokenKind t = keywords.get(name);
59 return (t != null) ? t : identifier;
60 }
61
62 /**
63 * This enum defines all tokens used by the javac scanner. A token is
64 * optionally associated with a name.
65 */
66 public enum TokenKind implements Predicate<TokenKind> {
67 EOF(),
68 ERROR(),
69 IDENTIFIER(Token.Tag.NAMED),
70 VALUE_IDENTIFIER(Token.Tag.NAMED),
71 INTLITERAL(Token.Tag.NUMERIC),
72 LONGLITERAL(Token.Tag.NUMERIC),
73 FLOATLITERAL(Token.Tag.NUMERIC),
74 DOUBLELITERAL(Token.Tag.NUMERIC),
75 CHARLITERAL(Token.Tag.NUMERIC),
76 STRINGLITERAL(Token.Tag.STRING),
77 TRUE("true", Token.Tag.NAMED),
78 FALSE("false", Token.Tag.NAMED),
79 NULL("null", Token.Tag.NAMED),
80 UNDERSCORE("_", Token.Tag.NAMED),
81 ARROW("->"),
82 LPAREN("("),
83 RPAREN(")"),
84 LBRACE("{"),
85 RBRACE("}"),
86 LBRACKET("["),
87 RBRACKET("]"),
88 COMMA(","),
89 DOT("."),
90 EQ("="),
91 GT(">"),
92 LT("<"),
93 QUES("?"),
94 COLON(":"),
95 COLCOL("::"),
96 SEMI(";"),
97 PLUS("+"),
98 SUB("-"),
99 AMP("&"),
100 CARET("^"),
101 MONKEYS_AT("@"),
102 HASH("#"),
103 CUSTOM;
104
105 public final String name;
106 public final Token.Tag tag;
107
108 TokenKind() {
109 this(null, Token.Tag.DEFAULT);
110 }
111
112 TokenKind(String name) {
113 this(name, Token.Tag.DEFAULT);
114 }
115
116 TokenKind(Token.Tag tag) {
117 this(null, tag);
118 }
119
120 TokenKind(String name, Token.Tag tag) {
121 this.name = name;
122 this.tag = tag;
123 }
124
125 public String toString() {
126 return switch (this) {
127 case IDENTIFIER -> "token.identifier";
128 case VALUE_IDENTIFIER -> "token.value-identifier";
129 case CHARLITERAL -> "token.character";
130 case STRINGLITERAL -> "token.string";
131 case INTLITERAL -> "token.integer";
132 case LONGLITERAL -> "token.long-integer";
133 case FLOATLITERAL -> "token.float";
134 case DOUBLELITERAL -> "token.double";
135 case ERROR -> "token.bad-symbol";
136 case EOF -> "token.end-of-input";
137 case DOT, COMMA, LPAREN, RPAREN, LBRACKET, RBRACKET, LBRACE, RBRACE -> "'" + name + "'";
138 default -> name;
139 };
140 }
141
142 @Override
143 public boolean test(TokenKind that) {
144 return this == that;
145 }
146 }
147
148 public interface Comment {
149
150 enum CommentStyle {
151 LINE,
152 BLOCK,
153 }
154
155 String text();
156
157 CommentStyle style();
158 }
159
160 /**
161 * This is the class representing a javac token. Each token has several fields
162 * that are set by the javac lexer (i.e. start/end position, string value, etc).
163 */
164 public static class Token {
165
166 /**
167 * tags constants
168 **/
169 public enum Tag {
170 DEFAULT,
171 NAMED,
172 STRING,
173 NUMERIC
174 }
175
176 /**
177 * The token kind
178 */
179 public final TokenKind kind;
180
181 /**
182 * The start position of this token
183 */
184 public final int pos;
185
186 /**
187 * The end position of this token
188 */
189 public final int endPos;
190
191 /**
192 * Comment reader associated with this token
193 */
194 public final List<Comment> comments;
195
196 Token(TokenKind kind, int pos, int endPos, List<Comment> comments) {
197 this.kind = kind;
198 this.pos = pos;
199 this.endPos = endPos;
200 this.comments = comments == null ? null : List.copyOf(comments);
201 checkKind();
202 }
203
204 void checkKind() {
205 if (kind.tag != Tag.DEFAULT) {
206 throw new AssertionError("Bad token kind - expected " + Tag.DEFAULT);
207 }
208 }
209
210 public String name() {
211 throw new UnsupportedOperationException();
212 }
213
214 public String stringVal() {
215 throw new UnsupportedOperationException();
216 }
217
218 public int radix() {
219 throw new UnsupportedOperationException();
220 }
221 }
222
223 static final class NamedToken extends Token {
224 /**
225 * The name of this token
226 */
227 public final String name;
228
229 NamedToken(TokenKind kind, int pos, int endPos, String name, List<Comment> comments) {
230 super(kind, pos, endPos, comments);
231 this.name = name;
232 }
233
234 void checkKind() {
235 if (kind.tag != Tag.NAMED) {
236 throw new AssertionError("Bad token kind - expected " + Tag.NAMED);
237 }
238 }
239
240 @Override
241 public String name() {
242 return name;
243 }
244 }
245
246 static class StringToken extends Token {
247 /**
248 * The string value of this token
249 */
250 public final String stringVal;
251
252 StringToken(TokenKind kind, int pos, int endPos, String stringVal, List<Comment> comments) {
253 super(kind, pos, endPos, comments);
254 this.stringVal = stringVal;
255 }
256
257 void checkKind() {
258 if (kind.tag != Tag.STRING) {
259 throw new AssertionError("Bad token kind - expected " + Tag.STRING);
260 }
261 }
262
263 @Override
264 public String stringVal() {
265 return stringVal;
266 }
267 }
268
269 static final class NumericToken extends StringToken {
270 /**
271 * The 'radix' value of this token
272 */
273 public final int radix;
274
275 NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, List<Comment> comments) {
276 super(kind, pos, endPos, stringVal, comments);
277 this.radix = radix;
278 }
279
280 void checkKind() {
281 if (kind.tag != Tag.NUMERIC) {
282 throw new AssertionError("Bad token kind - expected " + Tag.NUMERIC);
283 }
284 }
285
286 @Override
287 public int radix() {
288 return radix;
289 }
290 }
291
292 public static final Token DUMMY =
293 new Token(TokenKind.ERROR, 0, 0, null);
294 }