1 /*
   2  * Copyright (c) 1999, 2013, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package com.sun.tools.javac.parser;
  27 
  28 import java.util.Locale;
  29 
  30 import com.sun.tools.javac.api.Formattable;
  31 import com.sun.tools.javac.api.Messages;
  32 import com.sun.tools.javac.parser.Tokens.Token.Tag;
  33 import com.sun.tools.javac.util.List;
  34 import com.sun.tools.javac.util.Name;
  35 import com.sun.tools.javac.util.Context;
  36 import com.sun.tools.javac.util.Filter;
  37 import com.sun.tools.javac.util.ListBuffer;
  38 import com.sun.tools.javac.util.Names;
  39 
  40 /** A class that defines codes/utilities for Java source tokens
  41  *  returned from lexical analysis.
  42  *
  43  *  <p><b>This is NOT part of any supported API.
  44  *  If you write code that depends on this, you do so at your own risk.
  45  *  This code and its internal interfaces are subject to change or
  46  *  deletion without notice.</b>
  47  */
  48 public class Tokens {
  49 
  50     private final Names names;
  51 
  52     /**
  53      * Keyword array. Maps name indices to Token.
  54      */
  55     private final TokenKind[] key;
  56 
  57     /**  The number of the last entered keyword.
  58      */
  59     private int maxKey = 0;
  60 
  61     /** The names of all tokens.
  62      */
  63     private Name[] tokenName = new Name[TokenKind.values().length];
  64 
  65     public static final Context.Key<Tokens> tokensKey = new Context.Key<>();
  66 
  67     public static Tokens instance(Context context) {
  68         Tokens instance = context.get(tokensKey);
  69         if (instance == null)
  70             instance = new Tokens(context);
  71         return instance;
  72     }
  73 
  74     protected Tokens(Context context) {
  75         context.put(tokensKey, this);
  76         names = Names.instance(context);
  77         for (TokenKind t : TokenKind.values()) {
  78             if (t.name != null)
  79                 enterKeyword(t.name, t);
  80             else
  81                 tokenName[t.ordinal()] = null;
  82         }
  83 
  84         key = new TokenKind[maxKey+1];
  85         for (int i = 0; i <= maxKey; i++) key[i] = TokenKind.IDENTIFIER;
  86         for (TokenKind t : TokenKind.values()) {
  87             if (t.name != null)
  88             key[tokenName[t.ordinal()].getIndex()] = t;
  89         }
  90     }
  91 
  92     private void enterKeyword(String s, TokenKind token) {
  93         Name n = names.fromString(s);
  94         tokenName[token.ordinal()] = n;
  95         if (n.getIndex() > maxKey) maxKey = n.getIndex();
  96     }
  97 
  98     /**
  99      * Create a new token given a name; if the name corresponds to a token name,
 100      * a new token of the corresponding kind is returned; otherwise, an
 101      * identifier token is returned.
 102      */
 103     TokenKind lookupKind(Name name) {
 104         return (name.getIndex() > maxKey) ? TokenKind.IDENTIFIER : key[name.getIndex()];
 105     }
 106 
 107     TokenKind lookupKind(String name) {
 108         return lookupKind(names.fromString(name));
 109     }
 110 
 111     /**
 112      * This enum defines all tokens used by the javac scanner. A token is
 113      * optionally associated with a name.
 114      */
 115     public enum TokenKind implements Formattable, Filter<TokenKind> {
 116         EOF(),
 117         ERROR(),
 118         IDENTIFIER(Tag.NAMED),
 119         ABSTRACT("abstract"),
 120         ASSERT("assert", Tag.NAMED),
 121         BOOLEAN("boolean", Tag.NAMED),
 122         BREAK("break"),
 123         BYTE("byte", Tag.NAMED),
 124         CASE("case"),
 125         CATCH("catch"),
 126         CHAR("char", Tag.NAMED),
 127         CLASS("class"),
 128         CONST("const"),
 129         CONTINUE("continue"),
 130         DEFAULT("default"),
 131         DO("do"),
 132         DOUBLE("double", Tag.NAMED),
 133         ELSE("else"),
 134         ENUM("enum", Tag.NAMED),
 135         EXTENDS("extends"),
 136         FINAL("final"),
 137         FINALLY("finally"),
 138         FLOAT("float", Tag.NAMED),
 139         FOR("for"),
 140         GOTO("goto"),
 141         IF("if"),
 142         IMPLEMENTS("implements"),
 143         IMPORT("import"),
 144         INSTANCEOF("instanceof"),
 145         INT("int", Tag.NAMED),
 146         INTERFACE("interface"),
 147         LONG("long", Tag.NAMED),
 148         NATIVE("native"),
 149         NEW("new"),
 150         PACKAGE("package"),
 151         PRIVATE("private"),
 152         PROTECTED("protected"),
 153         PUBLIC("public"),
 154         RETURN("return"),
 155         SHORT("short", Tag.NAMED),
 156         STATIC("static"),
 157         STRICTFP("strictfp"),
 158         SUPER("super", Tag.NAMED),
 159         SWITCH("switch"),
 160         SYNCHRONIZED("synchronized"),
 161         THIS("this", Tag.NAMED),
 162         THROW("throw"),
 163         THROWS("throws"),
 164         TRANSIENT("transient"),
 165         TRY("try"),
 166         VALUE(), // a phantom token never returned by the scanner, but can result from a reclassification by the parser.
 167         VOID("void", Tag.NAMED),
 168         VOLATILE("volatile"),
 169         WHILE("while"),
 170         WITHFIELD("__WithField"),
 171         INTLITERAL(Tag.NUMERIC),
 172         LONGLITERAL(Tag.NUMERIC),
 173         FLOATLITERAL(Tag.NUMERIC),
 174         DOUBLELITERAL(Tag.NUMERIC),
 175         CHARLITERAL(Tag.NUMERIC),
 176         STRINGLITERAL(Tag.STRING),
 177         TRUE("true", Tag.NAMED),
 178         FALSE("false", Tag.NAMED),
 179         NULL("null", Tag.NAMED),
 180         UNDERSCORE("_", Tag.NAMED),
 181         ARROW("->"),
 182         COLCOL("::"),
 183         LPAREN("("),
 184         RPAREN(")"),
 185         LBRACE("{"),
 186         RBRACE("}"),
 187         LBRACKET("["),
 188         RBRACKET("]"),
 189         SEMI(";"),
 190         COMMA(","),
 191         DOT("."),
 192         ELLIPSIS("..."),
 193         EQ("="),
 194         GT(">"),
 195         LT("<"),
 196         BANG("!"),
 197         TILDE("~"),
 198         QUES("?"),
 199         COLON(":"),
 200         EQEQ("=="),
 201         LTEQ("<="),
 202         GTEQ(">="),
 203         BANGEQ("!="),
 204         AMPAMP("&&"),
 205         BARBAR("||"),
 206         PLUSPLUS("++"),
 207         SUBSUB("--"),
 208         PLUS("+"),
 209         SUB("-"),
 210         STAR("*"),
 211         SLASH("/"),
 212         AMP("&"),
 213         BAR("|"),
 214         CARET("^"),
 215         PERCENT("%"),
 216         LTLT("<<"),
 217         GTGT(">>"),
 218         GTGTGT(">>>"),
 219         PLUSEQ("+="),
 220         SUBEQ("-="),
 221         STAREQ("*="),
 222         SLASHEQ("/="),
 223         AMPEQ("&="),
 224         BAREQ("|="),
 225         CARETEQ("^="),
 226         PERCENTEQ("%="),
 227         LTLTEQ("<<="),
 228         GTGTEQ(">>="),
 229         GTGTGTEQ(">>>="),
 230         MONKEYS_AT("@"),
 231         CUSTOM;
 232 
 233         public final String name;
 234         public final Tag tag;
 235 
 236         TokenKind() {
 237             this(null, Tag.DEFAULT);
 238         }
 239 
 240         TokenKind(String name) {
 241             this(name, Tag.DEFAULT);
 242         }
 243 
 244         TokenKind(Tag tag) {
 245             this(null, tag);
 246         }
 247 
 248         TokenKind(String name, Tag tag) {
 249             this.name = name;
 250             this.tag = tag;
 251         }
 252 
 253         public String toString() {
 254             switch (this) {
 255             case IDENTIFIER:
 256                 return "token.identifier";
 257             case CHARLITERAL:
 258                 return "token.character";
 259             case STRINGLITERAL:
 260                 return "token.string";
 261             case INTLITERAL:
 262                 return "token.integer";
 263             case LONGLITERAL:
 264                 return "token.long-integer";
 265             case FLOATLITERAL:
 266                 return "token.float";
 267             case DOUBLELITERAL:
 268                 return "token.double";
 269             case ERROR:
 270                 return "token.bad-symbol";
 271             case VALUE:
 272                 return "value";
 273             case EOF:
 274                 return "token.end-of-input";
 275             case DOT: case COMMA: case SEMI: case LPAREN: case RPAREN:
 276             case LBRACKET: case RBRACKET: case LBRACE: case RBRACE:
 277                 return "'" + name + "'";
 278             default:
 279                 return name;
 280             }
 281         }
 282 
 283         public String getKind() {
 284             return "Token";
 285         }
 286 
 287         public String toString(Locale locale, Messages messages) {
 288             return name != null ? toString() : messages.getLocalizedString(locale, "compiler.misc." + toString());
 289         }
 290 
 291         @Override
 292         public boolean accepts(TokenKind that) {
 293             return this == that;
 294         }
 295     }
 296 
 297     public interface Comment {
 298 
 299         enum CommentStyle {
 300             LINE,
 301             BLOCK,
 302             JAVADOC,
 303         }
 304 
 305         String getText();
 306         int getSourcePos(int index);
 307         CommentStyle getStyle();
 308         boolean isDeprecated();
 309     }
 310 
 311     /**
 312      * This is the class representing a javac token. Each token has several fields
 313      * that are set by the javac lexer (i.e. start/end position, string value, etc).
 314      */
 315     public static class Token {
 316 
 317         /** tags constants **/
 318         enum Tag {
 319             DEFAULT,
 320             NAMED,
 321             STRING,
 322             NUMERIC
 323         }
 324 
 325         /** The token kind */
 326         public final TokenKind kind;
 327 
 328         /** The start position of this token */
 329         public final int pos;
 330 
 331         /** The end position of this token */
 332         public final int endPos;
 333 
 334         /** Comment reader associated with this token */
 335         public final List<Comment> comments;
 336 
 337         Token(TokenKind kind, int pos, int endPos, List<Comment> comments) {
 338             this.kind = kind;
 339             this.pos = pos;
 340             this.endPos = endPos;
 341             this.comments = comments;
 342             checkKind();
 343         }
 344 
 345         Token[] split(Tokens tokens) {
 346             if (kind.name.length() < 2 || kind.tag != Tag.DEFAULT) {
 347                 throw new AssertionError("Cant split" + kind);
 348             }
 349 
 350             TokenKind t1 = tokens.lookupKind(kind.name.substring(0, 1));
 351             TokenKind t2 = tokens.lookupKind(kind.name.substring(1));
 352 
 353             if (t1 == null || t2 == null) {
 354                 throw new AssertionError("Cant split - bad subtokens");
 355             }
 356             return new Token[] {
 357                 new Token(t1, pos, pos + t1.name.length(), comments),
 358                 new Token(t2, pos + t1.name.length(), endPos, null)
 359             };
 360         }
 361 
 362         protected void checkKind() {
 363             if (kind.tag != Tag.DEFAULT) {
 364                 throw new AssertionError("Bad token kind - expected " + Tag.STRING);
 365             }
 366         }
 367 
 368         public Name name() {
 369             throw new UnsupportedOperationException();
 370         }
 371 
 372         public String stringVal() {
 373             throw new UnsupportedOperationException();
 374         }
 375 
 376         public int radix() {
 377             throw new UnsupportedOperationException();
 378         }
 379 
 380         /**
 381          * Preserve classic semantics - if multiple javadocs are found on the token
 382          * the last one is returned
 383          */
 384         public Comment comment(Comment.CommentStyle style) {
 385             List<Comment> comments = getComments(Comment.CommentStyle.JAVADOC);
 386             return comments.isEmpty() ?
 387                     null :
 388                     comments.head;
 389         }
 390 
 391         /**
 392          * Preserve classic semantics - deprecated should be set if at least one
 393          * javadoc comment attached to this token contains the '@deprecated' string
 394          */
 395         public boolean deprecatedFlag() {
 396             for (Comment c : getComments(Comment.CommentStyle.JAVADOC)) {
 397                 if (c.isDeprecated()) {
 398                     return true;
 399                 }
 400             }
 401             return false;
 402         }
 403 
 404         private List<Comment> getComments(Comment.CommentStyle style) {
 405             if (comments == null) {
 406                 return List.nil();
 407             } else {
 408                 ListBuffer<Comment> buf = new ListBuffer<>();
 409                 for (Comment c : comments) {
 410                     if (c.getStyle() == style) {
 411                         buf.add(c);
 412                     }
 413                 }
 414                 return buf.toList();
 415             }
 416         }
 417     }
 418 
 419     final static class NamedToken extends Token {
 420         /** The name of this token */
 421         public final Name name;
 422 
 423         public NamedToken(TokenKind kind, int pos, int endPos, Name name, List<Comment> comments) {
 424             super(kind, pos, endPos, comments);
 425             this.name = name;
 426         }
 427 
 428         protected void checkKind() {
 429             if (kind.tag != Tag.NAMED) {
 430                 throw new AssertionError("Bad token kind - expected " + Tag.NAMED);
 431             }
 432         }
 433 
 434         @Override
 435         public Name name() {
 436             return name;
 437         }
 438     }
 439 
 440     static class StringToken extends Token {
 441         /** The string value of this token */
 442         public final String stringVal;
 443 
 444         public StringToken(TokenKind kind, int pos, int endPos, String stringVal, List<Comment> comments) {
 445             super(kind, pos, endPos, comments);
 446             this.stringVal = stringVal;
 447         }
 448 
 449         protected void checkKind() {
 450             if (kind.tag != Tag.STRING) {
 451                 throw new AssertionError("Bad token kind - expected " + Tag.STRING);
 452             }
 453         }
 454 
 455         @Override
 456         public String stringVal() {
 457             return stringVal;
 458         }
 459     }
 460 
 461     final static class NumericToken extends StringToken {
 462         /** The 'radix' value of this token */
 463         public final int radix;
 464 
 465         public NumericToken(TokenKind kind, int pos, int endPos, String stringVal, int radix, List<Comment> comments) {
 466             super(kind, pos, endPos, stringVal, comments);
 467             this.radix = radix;
 468         }
 469 
 470         protected void checkKind() {
 471             if (kind.tag != Tag.NUMERIC) {
 472                 throw new AssertionError("Bad token kind - expected " + Tag.NUMERIC);
 473             }
 474         }
 475 
 476         @Override
 477         public int radix() {
 478             return radix;
 479         }
 480     }
 481 
 482     public static final Token DUMMY =
 483                 new Token(TokenKind.ERROR, 0, 0, null);
 484 }