1 /*
   2  * Copyright (c) 1999, 2013, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package com.sun.tools.javac.parser;
  27 
  28 import java.util.Locale;
  29 
  30 import com.sun.tools.javac.api.Formattable;
  31 import com.sun.tools.javac.api.Messages;
  32 import com.sun.tools.javac.parser.Tokens.Token.Tag;
  33 import com.sun.tools.javac.util.List;
  34 import com.sun.tools.javac.util.Name;
  35 import com.sun.tools.javac.util.Context;
  36 import com.sun.tools.javac.util.Filter;
  37 import com.sun.tools.javac.util.ListBuffer;
  38 import com.sun.tools.javac.util.Names;
  39 
  40 /** A class that defines codes/utilities for Java source tokens
  41  *  returned from lexical analysis.
  42  *
  43  *  <p><b>This is NOT part of any supported API.
  44  *  If you write code that depends on this, you do so at your own risk.
  45  *  This code and its internal interfaces are subject to change or
  46  *  deletion without notice.</b>
  47  */
  48 public class Tokens {
  49 
  50     private final Names names;
  51 
  52     /**
  53      * Keyword array. Maps name indices to Token.
  54      */
  55     private final TokenKind[] key;
  56 
  57     /**  The number of the last entered keyword.
  58      */
  59     private int maxKey = 0;
  60 
  61     /** The names of all tokens.
  62      */
  63     private Name[] tokenName = new Name[TokenKind.values().length];
  64 
  65     public static final Context.Key<Tokens> tokensKey = new Context.Key<>();
  66 
  67     public static Tokens instance(Context context) {
  68         Tokens instance = context.get(tokensKey);
  69         if (instance == null)
  70             instance = new Tokens(context);
  71         return instance;
  72     }
  73 
  74     protected Tokens(Context context) {
  75         context.put(tokensKey, this);
  76         names = Names.instance(context);
  77         for (TokenKind t : TokenKind.values()) {
  78             if (t.name != null)
  79                 enterKeyword(t.name, t);
  80             else
  81                 tokenName[t.ordinal()] = null;
  82         }
  83 
  84         key = new TokenKind[maxKey+1];
  85         for (int i = 0; i <= maxKey; i++) key[i] = TokenKind.IDENTIFIER;
  86         for (TokenKind t : TokenKind.values()) {
  87             if (t.name != null && !t.reserved())
  88                 key[tokenName[t.ordinal()].getIndex()] = t;
  89         }
  90     }
  91 
  92     private void enterKeyword(String s, TokenKind token) {
  93         Name n = names.fromString(s);
  94         tokenName[token.ordinal()] = n;
  95         if (n.getIndex() > maxKey) maxKey = n.getIndex();
  96     }
  97 
  98     /**
  99      * Create a new token given a name; if the name corresponds to a token name,
 100      * a new token of the corresponding kind is returned; otherwise, an
 101      * identifier token is returned.
 102      */
 103     TokenKind lookupKind(Name name) {
 104         return (name.getIndex() > maxKey) ? TokenKind.IDENTIFIER : key[name.getIndex()];
 105     }
 106 
 107     TokenKind lookupKind(String name) {
 108         return lookupKind(names.fromString(name));
 109     }
 110 
 111     /**
 112      * This enum defines all tokens used by the javac scanner. A token is
 113      * optionally associated with a name.
 114      */
 115     public enum TokenKind implements Formattable, Filter<TokenKind> {
 116         EOF(),
 117         ERROR(),
 118         IDENTIFIER(Tag.NAMED),
 119         ABSTRACT("abstract"),
 120         ASSERT("assert", Tag.NAMED),
 121         BOOLEAN("boolean", Tag.NAMED),
 122         BREAK("break"),
 123         BYTE("byte", Tag.NAMED),
 124         CASE("case"),
 125         CATCH("catch"),
 126         CHAR("char", Tag.NAMED),
 127         CLASS("class"),
 128         CONST("const"),
 129         CONTINUE("continue"),
 130         DEFAULT("default"),
 131         DO("do"),
 132         DOUBLE("double", Tag.NAMED),
 133         ELSE("else"),
 134         ENUM("enum", Tag.NAMED),
 135         EXTENDS("extends"),
 136         FINAL("final"),
 137         FINALLY("finally"),
 138         FLOAT("float", Tag.NAMED),
 139         FOR("for"),
 140         GOTO("goto"),
 141         IF("if"),
 142         IMPLEMENTS("implements"),
 143         IMPORT("import"),
 144         INSTANCEOF("instanceof"),
 145         INT("int", Tag.NAMED),
 146         INTERFACE("interface"),
 147         LONG("long", Tag.NAMED),
 148         NATIVE("native"),
 149         NEW("new"),
 150         PACKAGE("package"),
 151         PRIVATE("private"),
 152         PROTECTED("protected"),
 153         PUBLIC("public"),
 154         RETURN("return"),
 155         SHORT("short", Tag.NAMED),
 156         STATIC("static"),
 157         STRICTFP("strictfp"),
 158         SUPER("super", Tag.NAMED),
 159         SWITCH("switch"),
 160         SYNCHRONIZED("synchronized"),
 161         THIS("this", Tag.NAMED),
 162         THROW("throw"),
 163         THROWS("throws"),
 164         TRANSIENT("transient"),
 165         TRY("try"),
 166         VOID("void", Tag.NAMED),
 167         VOLATILE("volatile"),
 168         WHILE("while"),
 169         INTLITERAL(Tag.NUMERIC),
 170         LONGLITERAL(Tag.NUMERIC),
 171         FLOATLITERAL(Tag.NUMERIC),
 172         DOUBLELITERAL(Tag.NUMERIC),
 173         CHARLITERAL(Tag.NUMERIC),
 174         STRINGLITERAL(Tag.STRING),
 175         TRUE("true", Tag.NAMED),
 176         FALSE("false", Tag.NAMED),
 177         NULL("null", Tag.NAMED),
 178         UNDERSCORE("_", Tag.NAMED),
 179         ARROW("->"),
 180         COLCOL("::"),
 181         LPAREN("("),
 182         RPAREN(")"),
 183         LBRACE("{"),
 184         RBRACE("}"),
 185         LBRACKET("["),
 186         RBRACKET("]"),
 187         SEMI(";"),
 188         COMMA(","),
 189         DOT("."),
 190         ELLIPSIS("..."),
 191         EQ("="),
 192         GT(">"),
 193         LT("<"),
 194         BANG("!"),
 195         TILDE("~"),
 196         QUES("?"),
 197         COLON(":"),
 198         EQEQ("=="),
 199         LTEQ("<="),
 200         GTEQ(">="),
 201         BANGEQ("!="),
 202         AMPAMP("&&"),
 203         BARBAR("||"),
 204         PLUSPLUS("++"),
 205         SUBSUB("--"),
 206         PLUS("+"),
 207         SUB("-"),
 208         STAR("*"),
 209         SLASH("/"),
 210         AMP("&"),
 211         BAR("|"),
 212         CARET("^"),
 213         PERCENT("%"),
 214         LTLT("<<"),
 215         GTGT(">>"),
 216         GTGTGT(">>>"),
 217         PLUSEQ("+="),
 218         SUBEQ("-="),
 219         STAREQ("*="),
 220         SLASHEQ("/="),
 221         AMPEQ("&="),
 222         BAREQ("|="),
 223         CARETEQ("^="),
 224         PERCENTEQ("%="),
 225         LTLTEQ("<<="),
 226         GTGTEQ(">>="),
 227         GTGTGTEQ(">>>="),
 228         MONKEYS_AT("@"),
 229         VAR("var", Tag.RESERVED),
 230         RECORD("record", Tag.RESERVED),
 231         SEALED("sealed", Tag.RESERVED),
 232         PERMITS("permits", Tag.RESERVED),
 233         CUSTOM;
 234 
 235         public final String name;
 236         public final Tag tag;
 237 
 238         TokenKind() {
 239             this(null, Tag.DEFAULT);
 240         }
 241 
 242         TokenKind(String name) {
 243             this(name, Tag.DEFAULT);
 244         }
 245 
 246         TokenKind(Tag tag) {
 247             this(null, tag);
 248         }
 249 
 250         TokenKind(String name, Tag tag) {
 251             this.name = name;
 252             this.tag = tag;
 253         }
 254 
 255         public String toString() {
 256             switch (this) {
 257             case IDENTIFIER:
 258                 return "token.identifier";
 259             case CHARLITERAL:
 260                 return "token.character";
 261             case STRINGLITERAL:
 262                 return "token.string";
 263             case INTLITERAL:
 264                 return "token.integer";
 265             case LONGLITERAL:
 266                 return "token.long-integer";
 267             case FLOATLITERAL:
 268                 return "token.float";
 269             case DOUBLELITERAL:
 270                 return "token.double";
 271             case ERROR:
 272                 return "token.bad-symbol";
 273             case EOF:
 274                 return "token.end-of-input";
 275             case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:
 276             case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:
 277                 return "'" + name + "'";
 278             default:
 279                 return name;
 280             }
 281         }
 282 
 283         public boolean reserved() {
 284             return tag == Tag.RESERVED;
 285         }
 286 
 287         public String getKind() {
 288             return "Token";
 289         }
 290 
 291         public String toString(Locale locale, Messages messages) {
 292             return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());
 293         }
 294 
 295         @Override
 296         public boolean accepts(TokenKind that) {
 297             return this == that;
 298         }
 299     }
 300 
 301     public interface Comment {
 302 
 303         enum CommentStyle {
 304             LINE,
 305             BLOCK,
 306             JAVADOC,
 307         }
 308 
 309         String getText();
 310         int getSourcePos(int index);
 311         CommentStyle getStyle();
 312         boolean isDeprecated();
 313     }
 314 
 315     /**
 316      * This is the class representing a javac token. Each token has several fields
 317      * that are set by the javac lexer (i.e. start/end position, string value, etc).
 318      */
 319     public static class Token {
 320 
 321         /** tags constants **/
 322         enum Tag {
 323             DEFAULT,
 324             NAMED,
 325             STRING,
 326             NUMERIC,
 327             RESERVED;
 328         }
 329 
 330         /** The token kind */
 331         public final TokenKind kind;
 332 
 333         /** The start position of this token */
 334         public final int pos;
 335 
 336         /** The end position of this token */
 337         public final int endPos;
 338 
 339         /** Comment reader associated with this token */
 340         public final List<Comment> comments;
 341 
 342         Token(TokenKind kind, int pos, int endPos, List<Comment> comments) {
 343             this.kind = kind;
 344             this.pos = pos;
 345             this.endPos = endPos;
 346             this.comments = comments;
 347             checkKind();
 348         }
 349 
 350         Token[] split(Tokens tokens) {
 351             if (kind.name.length() < 2 || kind.tag != Tag.DEFAULT) {
 352                 throw new AssertionError("Cant split" + kind);
 353             }
 354 
 355             TokenKind t1 = tokens.lookupKind(kind.name.substring(0, 1));
 356             TokenKind t2 = tokens.lookupKind(kind.name.substring(1));
 357 
 358             if (t1 == null || t2 == null) {
 359                 throw new AssertionError("Cant split - bad subtokens");
 360             }
 361             return new Token[] {
 362                 new Token(t1, pos, pos + t1.name.length(), comments),
 363                 new Token(t2, pos + t1.name.length(), endPos, null)
 364             };
 365         }
 366 
 367         protected void checkKind() {
 368             if (kind.tag != Tag.DEFAULT) {
 369                 throw new AssertionError("Bad token kind - expected " + Tag.STRING);
 370             }
 371         }
 372 
 373         public Name name() {
 374             throw new UnsupportedOperationException();
 375         }
 376 
 377         public String stringVal() {
 378             throw new UnsupportedOperationException();
 379         }
 380 
 381         public int radix() {
 382             throw new UnsupportedOperationException();
 383         }
 384 
 385         /**
 386          * Preserve classic semantics - if multiple javadocs are found on the token
 387          * the last one is returned
 388          */
 389         public Comment comment(Comment.CommentStyle style) {
 390             List<Comment> comments = getComments(Comment.CommentStyle.JAVADOC);
 391             return comments.isEmpty() ?
 392                     null :
 393                     comments.head;
 394         }
 395 
 396         /**
 397          * Preserve classic semantics - deprecated should be set if at least one
 398          * javadoc comment attached to this token contains the '@deprecated' string
 399          */
 400         public boolean deprecatedFlag() {
 401             for (Comment c : getComments(Comment.CommentStyle.JAVADOC)) {
 402                 if (c.isDeprecated()) {
 403                     return true;
 404                 }
 405             }
 406             return false;
 407         }
 408 
 409         private List<Comment> getComments(Comment.CommentStyle style) {
 410             if (comments == null) {
 411                 return List.nil();
 412             } else {
 413                 ListBuffer<Comment> buf = new ListBuffer<>();
 414                 for (Comment c : comments) {
 415                     if (c.getStyle() == style) {
 416                         buf.add(c);
 417                     }
 418                 }
 419                 return buf.toList();
 420             }
 421         }
 422     }
 423 
 424     final static class NamedToken extends Token {
 425         /** The name of this token */
 426         public final Name name;
 427 
 428         public NamedToken(TokenKind kind, int pos, int endPos, Name name, List<Comment> comments) {
 429             super(kind, pos, endPos, comments);
 430             this.name = name;
 431         }
 432 
 433         protected void checkKind() {
 434             if (kind.tag != Tag.NAMED) {
 435                 throw new AssertionError("Bad token kind - expected " + Tag.NAMED);
 436             }
 437         }
 438 
 439         @Override
 440         public Name name() {
 441             return name;
 442         }
 443     }
 444 
 445     static class StringToken extends Token {
 446         /** The string value of this token */
 447         public final String stringVal;
 448 
 449         public StringToken(TokenKind kind, int pos, int endPos, String stringVal, List<Comment> comments) {
 450             super(kind, pos, endPos, comments);
 451             this.stringVal = stringVal;
 452         }
 453 
 454         protected void checkKind() {
 455             if (kind.tag != Tag.STRING) {
 456                 throw new AssertionError("Bad token kind - expected " + Tag.STRING);
 457             }
 458         }
 459 
 460         @Override
 461         public String stringVal() {
 462             return stringVal;
 463         }
 464     }
 465 
 466     final static class NumericToken extends StringToken {
 467         /** The 'radix' value of this token */
 468         public final int radix;
 469 
 470         public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, List<Comment> comments) {
 471             super(kind, pos, endPos, stringVal, comments);
 472             this.radix = radix;
 473         }
 474 
 475         protected void checkKind() {
 476             if (kind.tag != Tag.NUMERIC) {
 477                 throw new AssertionError("Bad token kind - expected " + Tag.NUMERIC);
 478             }
 479         }
 480 
 481         @Override
 482         public int radix() {
 483             return radix;
 484         }
 485     }
 486 
 487     public static final Token DUMMY =
 488                 new Token(TokenKind.ERROR, 0, 0, null);
 489 }