1 /*
   2  * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package jdk.jshell;
  27 
  28 import com.sun.tools.javac.code.Source;
  29 import com.sun.tools.javac.parser.Scanner;
  30 import com.sun.tools.javac.parser.ScannerFactory;
  31 import com.sun.tools.javac.parser.Tokens.Token;
  32 import com.sun.tools.javac.parser.Tokens.TokenKind;
  33 import com.sun.tools.javac.util.Context;
  34 import com.sun.tools.javac.util.DiagnosticSource;
  35 import com.sun.tools.javac.util.JCDiagnostic;
  36 import com.sun.tools.javac.util.JCDiagnostic.DiagnosticFlag;
  37 import com.sun.tools.javac.util.JCDiagnostic.DiagnosticPosition;
  38 import com.sun.tools.javac.util.JCDiagnostic.Error;
  39 import com.sun.tools.javac.util.Log;
  40 import java.io.PrintWriter;
  41 import java.io.StringWriter;
  42 import java.util.ArrayDeque;
  43 import java.util.Deque;
  44 import java.util.EnumMap;
  45 import java.util.Iterator;
  46 import jdk.jshell.SourceCodeAnalysis.Completeness;
  47 import com.sun.source.tree.Tree;
  48 import static jdk.jshell.CompletenessAnalyzer.TK.*;
  49 import jdk.jshell.TaskFactory.ParseTask;
  50 import jdk.jshell.TaskFactory.Worker;
  51 import java.util.List;
  52 import java.util.function.Function;
  53 import java.util.function.Supplier;
  54 
  55 /**
  56  * Low level scanner to determine completeness of input.
  57  * @author Robert Field
  58  */
  59 class CompletenessAnalyzer {
  60 
  61     private final ScannerFactory scannerFactory;
  62     private final JShell proc;
  63 
  64     private static Completeness error() {
  65         return Completeness.UNKNOWN;  // For breakpointing
  66     }
  67 
  68     static class CaInfo {
  69 
  70         CaInfo(Completeness status, int unitEndPos) {
  71             this.status = status;
  72             this.unitEndPos = unitEndPos;
  73         }
  74         final int unitEndPos;
  75         final Completeness status;
  76     }
  77 
  78     CompletenessAnalyzer(JShell proc) {
  79         this.proc = proc;
  80         Context context = new Context();
  81         Log log = CaLog.createLog(context);
  82         context.put(Log.class, log);
  83         context.put(Source.class, Source.JDK9);
  84         scannerFactory = ScannerFactory.instance(context);
  85     }
  86 
  87     CaInfo scan(String s) {
  88         try {
  89             Parser parser = new Parser(
  90                     () -> new Matched(scannerFactory.newScanner(s, false)),
  91                     worker -> proc.taskFactory.parse(s, worker));
  92             Completeness stat = parser.parseUnit();
  93             int endPos = stat == Completeness.UNKNOWN
  94                     ? s.length()
  95                     : parser.endPos();
  96             return new CaInfo(stat, endPos);
  97         } catch (SyntaxException ex) {
  98             return new CaInfo(error(), s.length());
  99         }
 100     }
 101 
 102     @SuppressWarnings("serial")             // serialVersionUID intentionally omitted
 103     private static class SyntaxException extends RuntimeException {
 104     }
 105 
 106     private static void die() {
 107         throw new SyntaxException();
 108     }
 109 
 110     /**
 111      * Subclass of Log used by compiler API to die on error and ignore
 112      * other messages
 113      */
 114     private static class CaLog extends Log {
 115 
 116         private static CaLog createLog(Context context) {
 117             PrintWriter pw = new PrintWriter(new StringWriter());
 118             CaLog log = new CaLog(context, pw);
 119             context.put(logKey, log);
 120             return log;
 121         }
 122 
 123         private CaLog(Context context, PrintWriter pw) {
 124             super(context, pw);
 125             this.source = DiagnosticSource.NO_SOURCE;
 126         }
 127 
 128         @Override
 129         public void error(String key, Object... args) {
 130             die();
 131         }
 132 
 133         @Override
 134         public void error(int pos, Error errorKey) {
 135             die();
 136         }
 137 
 138         @Override
 139         public void error(int pos, String key, Object... args) {
 140             die();
 141         }
 142 
 143         @Override
 144         public void report(JCDiagnostic diagnostic) {
 145             // Ignore
 146         }
 147     }
 148 
 149     // Location position kinds -- a token is ...
 150     private static final int XEXPR         = 0b1;                       // OK in expression (not first)
 151     private static final int XDECL         = 0b10;                      // OK in declaration (not first)
 152     private static final int XSTMT         = 0b100;                     // OK in statement framework (not first)
 153     private static final int XEXPR1o       = 0b1000;                    // OK first in expression
 154     private static final int XDECL1o       = 0b10000;                   // OK first in declaration
 155     private static final int XSTMT1o       = 0b100000;                  // OK first or only in statement framework
 156     private static final int XEXPR1        = XEXPR1o | XEXPR;           // OK in expression (anywhere)
 157     private static final int XDECL1        = XDECL1o | XDECL;           // OK in declaration (anywhere)
 158     private static final int XSTMT1        = XSTMT1o | XSTMT;           // OK in statement framework (anywhere)
 159     private static final int XANY1         = XEXPR1o | XDECL1o | XSTMT1o;  // Mask: first in statement, declaration, or expression
 160     private static final int XTERM         = 0b100000000;               // Can terminate (last before EOF)
 161     private static final int XSTART        = 0b1000000000;              // Boundary, must be XTERM before
 162     private static final int XERRO         = 0b10000000000;             // Is an error
 163     private static final int XBRACESNEEDED = 0b100000000000;            // Expect {ANY} LBRACE
 164 
 165     /**
 166      * An extension of the compiler's TokenKind which adds our combined/processed
 167      * kinds. Also associates each TK with a union of acceptable kinds of code
 168      * position it can occupy.  For example: IDENTIFER is XEXPR1|XDECL1|XTERM,
 169      * meaning it can occur in expressions or declarations (but not in the
 170      * framework of a statement and that can be the final (terminating) token
 171      * in a snippet.
 172      * <P>
 173      * There must be a TK defined for each compiler TokenKind, an exception
 174      * will
 175      * be thrown if a TokenKind is defined and a corresponding TK is not. Add a
 176      * new TK in the appropriate category. If it is like an existing category
 177      * (e.g. a new modifier or type this may be all that is needed.  If it
 178      * is bracketing or modifies the acceptable positions of other tokens,
 179      * please closely examine the needed changes to this scanner.
 180      */
 181     static enum TK {
 182 
 183         // Special
 184         EOF(TokenKind.EOF, 0),  //
 185         ERROR(TokenKind.ERROR, XERRO),  //
 186         IDENTIFIER(TokenKind.IDENTIFIER, XEXPR1|XDECL1|XTERM),  //
 187         UNDERSCORE(TokenKind.UNDERSCORE, XERRO),  //  _
 188         CLASS(TokenKind.CLASS, XEXPR|XDECL1|XBRACESNEEDED),  //  class decl (MAPPED: DOTCLASS)
 189         MONKEYS_AT(TokenKind.MONKEYS_AT, XEXPR|XDECL1),  //  @
 190         IMPORT(TokenKind.IMPORT, XDECL1|XSTART),  //  import -- consider declaration
 191         SEMI(TokenKind.SEMI, XSTMT1|XTERM|XSTART),  //  ;
 192 
 193         // Shouldn't see -- error
 194         PACKAGE(TokenKind.PACKAGE, XERRO),  //  package
 195         CONST(TokenKind.CONST, XERRO),  //  reserved keyword -- const
 196         GOTO(TokenKind.GOTO, XERRO),  //  reserved keyword -- goto
 197         CUSTOM(TokenKind.CUSTOM, XERRO),  // No uses
 198 
 199         // Declarations
 200         ENUM(TokenKind.ENUM, XDECL1|XBRACESNEEDED),  //  enum
 201         IMPLEMENTS(TokenKind.IMPLEMENTS, XDECL),  //  implements
 202         INTERFACE(TokenKind.INTERFACE, XDECL1|XBRACESNEEDED),  //  interface
 203         THROWS(TokenKind.THROWS, XDECL|XBRACESNEEDED),  //  throws
 204 
 205         // Primarive type names
 206         BOOLEAN(TokenKind.BOOLEAN, XEXPR1|XDECL1),  //  boolean
 207         BYTE(TokenKind.BYTE, XEXPR1|XDECL1),  //  byte
 208         CHAR(TokenKind.CHAR, XEXPR1|XDECL1),  //  char
 209         DOUBLE(TokenKind.DOUBLE, XEXPR1|XDECL1),  //  double
 210         FLOAT(TokenKind.FLOAT, XEXPR1|XDECL1),  //  float
 211         INT(TokenKind.INT, XEXPR1|XDECL1),  //  int
 212         LONG(TokenKind.LONG, XEXPR1|XDECL1),  //  long
 213         SHORT(TokenKind.SHORT, XEXPR1|XDECL1),  //  short
 214         VOID(TokenKind.VOID, XEXPR1|XDECL1),  //  void
 215 
 216         // Modifiers keywords
 217         ABSTRACT(TokenKind.ABSTRACT, XDECL1),  //  abstract
 218         FINAL(TokenKind.FINAL, XDECL1),  //  final
 219         NATIVE(TokenKind.NATIVE, XDECL1),  //  native
 220         STATIC(TokenKind.STATIC, XDECL1),  //  static
 221         STRICTFP(TokenKind.STRICTFP, XDECL1),  //  strictfp
 222         PRIVATE(TokenKind.PRIVATE, XDECL1),  //  private
 223         PROTECTED(TokenKind.PROTECTED, XDECL1),  //  protected
 224         PUBLIC(TokenKind.PUBLIC, XDECL1),  //  public
 225         TRANSIENT(TokenKind.TRANSIENT, XDECL1),  //  transient
 226         VOLATILE(TokenKind.VOLATILE, XDECL1),  //  volatile
 227         VALUE(TokenKind.VALUE, 0),
 228 
 229         // Declarations and type parameters (thus expressions)
 230         EXTENDS(TokenKind.EXTENDS, XEXPR|XDECL),  //  extends
 231         COMMA(TokenKind.COMMA, XEXPR|XDECL),  //  ,
 232         AMP(TokenKind.AMP, XEXPR|XDECL, true),  //  &
 233         GT(TokenKind.GT, XEXPR|XDECL, true),  //  >
 234         LT(TokenKind.LT, XEXPR|XDECL1, true),  //  <
 235         LTLT(TokenKind.LTLT, XEXPR|XDECL1, true),  //  <<
 236         GTGT(TokenKind.GTGT, XEXPR|XDECL, true),  //  >>
 237         GTGTGT(TokenKind.GTGTGT, XEXPR|XDECL, true),  //  >>>
 238         QUES(TokenKind.QUES, XEXPR|XDECL, true),  //  ?
 239         DOT(TokenKind.DOT, XEXPR|XDECL),  //  .
 240         STAR(TokenKind.STAR, XEXPR, true),  //  * (MAPPED: DOTSTAR)
 241 
 242         // Statement keywords
 243         ASSERT(TokenKind.ASSERT, XSTMT1|XSTART),  //  assert
 244         BREAK(TokenKind.BREAK, XSTMT1|XTERM|XSTART),  //  break
 245         CATCH(TokenKind.CATCH, XSTMT1|XSTART),  //  catch
 246         CONTINUE(TokenKind.CONTINUE, XSTMT1|XTERM|XSTART),  //  continue
 247         DO(TokenKind.DO, XSTMT1|XSTART),  //  do
 248         ELSE(TokenKind.ELSE, XSTMT1|XTERM|XSTART),  //  else
 249         FINALLY(TokenKind.FINALLY, XSTMT1|XSTART),  //  finally
 250         FOR(TokenKind.FOR, XSTMT1|XSTART),  //  for
 251         IF(TokenKind.IF, XSTMT1|XSTART),  //  if
 252         RETURN(TokenKind.RETURN, XSTMT1|XTERM|XSTART),  //  return
 253         SWITCH(TokenKind.SWITCH, XSTMT1|XEXPR1),  //  switch
 254         SYNCHRONIZED(TokenKind.SYNCHRONIZED, XSTMT1|XDECL),  //  synchronized
 255         THROW(TokenKind.THROW, XSTMT1|XSTART),  //  throw
 256         TRY(TokenKind.TRY, XSTMT1|XSTART),  //  try
 257         WHILE(TokenKind.WHILE, XSTMT1|XSTART),  //  while
 258         WITHFIELD(TokenKind.WITHFIELD, XEXPR),  //  withfield
 259 
 260         // Statement keywords that we shouldn't see -- inside braces
 261         CASE(TokenKind.CASE, XSTMT|XSTART),  //  case
 262         DEFAULT(TokenKind.DEFAULT, XSTMT|XSTART),  //  default method, default case -- neither we should see
 263 
 264         // Expressions (can terminate)
 265         INTLITERAL(TokenKind.INTLITERAL, XEXPR1|XTERM),  //
 266         LONGLITERAL(TokenKind.LONGLITERAL, XEXPR1|XTERM),  //
 267         FLOATLITERAL(TokenKind.FLOATLITERAL, XEXPR1|XTERM),  //
 268         DOUBLELITERAL(TokenKind.DOUBLELITERAL, XEXPR1|XTERM),  //
 269         CHARLITERAL(TokenKind.CHARLITERAL, XEXPR1|XTERM),  //
 270         STRINGLITERAL(TokenKind.STRINGLITERAL, XEXPR1|XTERM),  //
 271         TRUE(TokenKind.TRUE, XEXPR1|XTERM),  //  true
 272         FALSE(TokenKind.FALSE, XEXPR1|XTERM),  //  false
 273         NULL(TokenKind.NULL, XEXPR1|XTERM),  //  null
 274         THIS(TokenKind.THIS, XEXPR1|XTERM),  //  this  -- shouldn't see
 275 
 276         // Expressions maybe terminate  //TODO handle these case separately
 277         PLUSPLUS(TokenKind.PLUSPLUS, XEXPR1|XTERM),  //  ++
 278         SUBSUB(TokenKind.SUBSUB, XEXPR1|XTERM),  //  --
 279 
 280         // Expressions cannot terminate
 281         INSTANCEOF(TokenKind.INSTANCEOF, XEXPR, true),  //  instanceof
 282         NEW(TokenKind.NEW, XEXPR1),  //  new (MAPPED: COLCOLNEW)
 283         SUPER(TokenKind.SUPER, XEXPR1|XDECL),  //  super -- shouldn't see as rec. But in type parameters
 284         ARROW(TokenKind.ARROW, XEXPR),  //  ->
 285         COLCOL(TokenKind.COLCOL, XEXPR),  //  ::
 286         LPAREN(TokenKind.LPAREN, XEXPR),  //  (
 287         RPAREN(TokenKind.RPAREN, XEXPR),  //  )
 288         LBRACE(TokenKind.LBRACE, XEXPR),  //  {
 289         RBRACE(TokenKind.RBRACE, XEXPR),  //  }
 290         LBRACKET(TokenKind.LBRACKET, XEXPR),  //  [
 291         RBRACKET(TokenKind.RBRACKET, XEXPR),  //  ]
 292         ELLIPSIS(TokenKind.ELLIPSIS, XEXPR),  //  ...
 293         EQ(TokenKind.EQ, XEXPR),  //  =
 294         BANG(TokenKind.BANG, XEXPR1),  //  !
 295         TILDE(TokenKind.TILDE, XEXPR1),  //  ~
 296         COLON(TokenKind.COLON, XEXPR|XTERM),  //  :
 297         EQEQ(TokenKind.EQEQ, XEXPR, true),  //  ==
 298         LTEQ(TokenKind.LTEQ, XEXPR, true),  //  <=
 299         GTEQ(TokenKind.GTEQ, XEXPR, true),  //  >=
 300         BANGEQ(TokenKind.BANGEQ, XEXPR, true),  //  !=
 301         AMPAMP(TokenKind.AMPAMP, XEXPR, true),  //  &&
 302         BARBAR(TokenKind.BARBAR, XEXPR, true),  //  ||
 303         PLUS(TokenKind.PLUS, XEXPR1, true),  //  +
 304         SUB(TokenKind.SUB, XEXPR1, true),  //  -
 305         SLASH(TokenKind.SLASH, XEXPR, true),  //  /
 306         BAR(TokenKind.BAR, XEXPR, true),  //  |
 307         CARET(TokenKind.CARET, XEXPR, true),  //  ^
 308         PERCENT(TokenKind.PERCENT, XEXPR, true),  //  %
 309         PLUSEQ(TokenKind.PLUSEQ, XEXPR),  //  +=
 310         SUBEQ(TokenKind.SUBEQ, XEXPR),  //  -=
 311         STAREQ(TokenKind.STAREQ, XEXPR),  //  *=
 312         SLASHEQ(TokenKind.SLASHEQ, XEXPR),  //  /=
 313         AMPEQ(TokenKind.AMPEQ, XEXPR),  //  &=
 314         BAREQ(TokenKind.BAREQ, XEXPR),  //  |=
 315         CARETEQ(TokenKind.CARETEQ, XEXPR),  //  ^=
 316         PERCENTEQ(TokenKind.PERCENTEQ, XEXPR),  //  %=
 317         LTLTEQ(TokenKind.LTLTEQ, XEXPR),  //  <<=
 318         GTGTEQ(TokenKind.GTGTEQ, XEXPR),  //  >>=
 319         GTGTGTEQ(TokenKind.GTGTGTEQ, XEXPR),  //  >>>=
 320 
 321         // combined/processed kinds
 322         UNMATCHED(XERRO),
 323         PARENS(XEXPR1|XDECL|XSTMT|XTERM),
 324         BRACKETS(XEXPR|XDECL|XTERM),
 325         BRACES(XSTMT1|XEXPR|XTERM),
 326         DOTSTAR(XDECL|XTERM),  // import foo.*
 327         COLCOLNEW(XEXPR|XTERM),  //  :: new
 328         DOTCLASS(XEXPR|XTERM),  //  class decl and .class
 329         ;
 330 
 331         static final EnumMap<TokenKind,TK> tokenKindToTKMap = new EnumMap<>(TokenKind.class);
 332 
 333         final TokenKind tokenKind;
 334         final int belongs;
 335         final boolean valueOp;
 336         Function<TK,TK> mapping;
 337 
 338         TK(int b) {
 339             this(null, b);
 340         }
 341 
 342         TK(TokenKind tokenKind, int b) {
 343             this(tokenKind, b, false);
 344         }
 345 
 346         TK(TokenKind tokenKind, int b, boolean valueOp) {
 347             this.tokenKind = tokenKind;
 348             this.belongs = b;
 349             this.valueOp = valueOp;
 350             this.mapping = null;
 351         }
 352 
 353         private static TK tokenKindToTK(TK prev, TokenKind kind) {
 354             TK tk = tokenKindToTKMap.get(kind);
 355             if (tk == null) {
 356                 System.err.printf("No corresponding %s for %s: %s\n",
 357                         TK.class.getCanonicalName(),
 358                         TokenKind.class.getCanonicalName(),
 359                         kind);
 360                 throw new InternalError("No corresponding TK for TokenKind: " + kind);
 361             }
 362             return tk.mapping != null
 363                     ? tk.mapping.apply(prev)
 364                     : tk;
 365         }
 366 
 367         boolean isOkToTerminate() {
 368             return (belongs & XTERM) != 0;
 369         }
 370 
 371         boolean isExpression() {
 372             return (belongs & XEXPR) != 0;
 373         }
 374 
 375         boolean isDeclaration() {
 376             return (belongs & XDECL) != 0;
 377         }
 378 
 379         boolean isError() {
 380             return (belongs & XERRO) != 0;
 381         }
 382 
 383         boolean isStart() {
 384             return (belongs & XSTART) != 0;
 385         }
 386 
 387         boolean isBracesNeeded() {
 388             return (belongs & XBRACESNEEDED) != 0;
 389         }
 390 
 391         /**
 392          * After construction, check that all compiler TokenKind values have
 393          * corresponding TK values.
 394          */
 395         static {
 396             for (TK tk : TK.values()) {
 397                 if (tk.tokenKind != null) {
 398                     tokenKindToTKMap.put(tk.tokenKind, tk);
 399                 }
 400             }
 401             for (TokenKind kind : TokenKind.values()) {
 402                 tokenKindToTK(null, kind); // assure they can be retrieved without error
 403             }
 404             // Mappings of disambiguated contexts
 405             STAR.mapping  = prev -> prev == DOT ? DOTSTAR : STAR;
 406             NEW.mapping   = prev -> prev == COLCOL ? COLCOLNEW : NEW;
 407             CLASS.mapping = prev -> prev == DOT ? DOTCLASS : CLASS;
 408         }
 409     }
 410 
 411     /**
 412      * A completeness scanner token.
 413      */
 414     private static class CT {
 415 
 416         /** The token kind */
 417         public final TK kind;
 418 
 419         /** The end position of this token */
 420         public final int endPos;
 421 
 422         /** The error message **/
 423         public final String message;
 424 
 425         private CT(TK tk, Token tok, String msg) {
 426             this.kind = tk;
 427             this.endPos = tok.endPos;
 428             this.message = msg;
 429             //throw new InternalError(msg); /* for debugging */
 430         }
 431 
 432         private CT(TK tk, Token tok) {
 433             this.kind = tk;
 434             this.endPos = tok.endPos;
 435             this.message = null;
 436         }
 437 
 438         private CT(TK tk, int endPos) {
 439             this.kind = tk;
 440             this.endPos = endPos;
 441             this.message = null;
 442         }
 443     }
 444 
 445     /**
 446      * Look for matching tokens (like parens) and other special cases, like "new"
 447      */
 448     private static class Matched implements Iterator<CT> {
 449 
 450         private final Scanner scanner;
 451         private Token current;
 452         private CT prevCT;
 453         private CT currentCT;
 454         private final Deque<Token> stack = new ArrayDeque<>();
 455 
 456         Matched(Scanner scanner) {
 457             this.scanner = scanner;
 458             advance();
 459             prevCT = currentCT = new CT(SEMI, 0); // So is valid for testing
 460         }
 461 
 462         @Override
 463         public boolean hasNext() {
 464             return currentCT.kind != EOF;
 465         }
 466 
 467         private Token advance() {
 468             Token prev = current;
 469             scanner.nextToken();
 470             current = scanner.token();
 471             return prev;
 472         }
 473 
 474         @Override
 475         public CT next() {
 476             prevCT = currentCT;
 477             currentCT = nextCT();
 478             return currentCT;
 479         }
 480 
 481         private CT match(TK tk, TokenKind open) {
 482             Token tok = advance();
 483             db("match desired-tk=%s, open=%s, seen-tok=%s", tk, open, tok.kind);
 484             if (stack.isEmpty()) {
 485                 return new CT(ERROR, tok, "Encountered '" + tok + "' with no opening '" + open + "'");
 486             }
 487             Token p = stack.pop();
 488             if (p.kind != open) {
 489                 return new CT(ERROR, tok, "No match for '" + p + "' instead encountered '" + tok + "'");
 490             }
 491             return new CT(tk, tok);
 492         }
 493 
 494         private void db(String format, Object ... args) {
 495 //            System.err.printf(format, args);
 496 //            System.err.printf(" -- stack(");
 497 //            if (stack.isEmpty()) {
 498 //
 499 //            } else {
 500 //                for (Token tok : stack) {
 501 //                    System.err.printf("%s ", tok.kind);
 502 //                }
 503 //            }
 504 //            System.err.printf(") current=%s / currentCT=%s\n", current.kind, currentCT.kind);
 505         }
 506 
 507         /**
 508          * @return the next scanner token
 509          */
 510         private CT nextCT() {
 511             // TODO Annotations?
 512             TK prevTK = currentCT.kind;
 513             while (true) {
 514                 db("nextCT");
 515                 CT ct;
 516                 switch (current.kind) {
 517                     case EOF:
 518                         db("eof");
 519                         if (stack.isEmpty()) {
 520                             ct = new CT(EOF, current);
 521                         } else {
 522                             TokenKind unmatched = stack.pop().kind;
 523                             stack.clear(); // So we will get EOF next time
 524                             ct = new CT(UNMATCHED, current, "Unmatched " + unmatched);
 525                         }
 526                         break;
 527                     case LPAREN:
 528                     case LBRACE:
 529                     case LBRACKET:
 530                         stack.push(advance());
 531                         prevTK = SEMI; // new start
 532                         continue;
 533                     case RPAREN:
 534                         ct = match(PARENS, TokenKind.LPAREN);
 535                         break;
 536                     case RBRACE:
 537                         ct = match(BRACES, TokenKind.LBRACE);
 538                         break;
 539                     case RBRACKET:
 540                         ct = match(BRACKETS, TokenKind.LBRACKET);
 541                         break;
 542                     default:
 543                         ct = new CT(TK.tokenKindToTK(prevTK, current.kind), advance());
 544                         break;
 545                 }
 546                 // Detect an error if we are at starting position and the last
 547                 // token wasn't a terminating one.  Special case: within braces,
 548                 // comma can proceed semicolon, e.g. the values list in enum
 549                 if (ct.kind.isStart() && !prevTK.isOkToTerminate() && prevTK != COMMA) {
 550                     return new CT(ERROR, current, "No '" + prevTK + "' before '" + ct.kind + "'");
 551                 }
 552                 if (stack.isEmpty() || ct.kind.isError()) {
 553                     return ct;
 554                 }
 555                 prevTK = ct.kind;
 556             }
 557         }
 558     }
 559 
 560     /**
 561      * Fuzzy parser based on token kinds
 562      */
 563     private static class Parser {
 564 
 565         private final Supplier<Matched> matchedFactory;
 566         private final Function<Worker<ParseTask, Completeness>, Completeness> parseFactory;
 567         private Matched in;
 568         private CT token;
 569         private Completeness checkResult;
 570 
 571         Parser(Supplier<Matched> matchedFactory,
 572                Function<Worker<ParseTask, Completeness>, Completeness> parseFactory) {
 573             this.matchedFactory = matchedFactory;
 574             this.parseFactory = parseFactory;
 575             resetInput();
 576         }
 577 
 578         final void resetInput() {
 579             this.in = matchedFactory.get();
 580             nextToken();
 581         }
 582 
 583         final void nextToken() {
 584             in.next();
 585             token = in.currentCT;
 586         }
 587 
 588         boolean shouldAbort(TK tk) {
 589             if (token.kind == tk) {
 590                 nextToken();
 591                 return false;
 592             }
 593             switch (token.kind) {
 594                 case EOF:
 595                     checkResult = ((tk == SEMI) && in.prevCT.kind.isOkToTerminate())
 596                             ? Completeness.COMPLETE_WITH_SEMI
 597                             : Completeness.DEFINITELY_INCOMPLETE;
 598                     return true;
 599                 case UNMATCHED:
 600                     checkResult = Completeness.DEFINITELY_INCOMPLETE;
 601                     return true;
 602                 default:
 603                     checkResult = error();
 604                     return true;
 605 
 606             }
 607         }
 608 
 609         Completeness lastly(TK tk) {
 610             if (shouldAbort(tk))  return checkResult;
 611             return Completeness.COMPLETE;
 612         }
 613 
 614         Completeness optionalFinalSemi() {
 615             if (!shouldAbort(SEMI)) return Completeness.COMPLETE;
 616             if (checkResult == Completeness.COMPLETE_WITH_SEMI) return Completeness.COMPLETE;
 617             return checkResult;
 618         }
 619 
 620         boolean shouldAbort(Completeness flags) {
 621             checkResult = flags;
 622             return flags != Completeness.COMPLETE;
 623         }
 624 
 625         public int endPos() {
 626             return in.prevCT.endPos;
 627         }
 628 
 629         public Completeness parseUnit() {
 630             //System.err.printf("%s:  belongs %o  XANY1 %o\n", token.kind, token.kind.belongs, token.kind.belongs & XANY1);
 631             switch (token.kind.belongs & XANY1) {
 632                 case XEXPR1o:
 633                     return parseExpressionOptionalSemi();
 634                 case XSTMT1o: {
 635                     Completeness stat = parseSimpleStatement();
 636                     return stat==null? error() : stat;
 637                 }
 638                 case XDECL1o:
 639                     return parseDeclaration();
 640                 case XSTMT1o | XDECL1o:
 641                 case XEXPR1o | XDECL1o:
 642                     return disambiguateDeclarationVsExpression();
 643                 case 0:
 644                     if ((token.kind.belongs & XERRO) != 0) {
 645                         return parseExpressionStatement(); // Let this gen the status
 646                     }
 647                     return error();
 648                 case XSTMT1o | XEXPR1o:
 649                     return disambiguateStatementVsExpression();
 650                 default:
 651                     throw new InternalError("Case not covered " + token.kind.belongs + " in " + token.kind);
 652             }
 653         }
 654 
 655         public Completeness parseDeclaration() {
 656             boolean isImport = token.kind == IMPORT;
 657             boolean isBracesNeeded = false;
 658             while (token.kind.isDeclaration()) {
 659                 isBracesNeeded |= token.kind.isBracesNeeded();
 660                 nextToken();
 661             }
 662             switch (token.kind) {
 663                 case EQ:
 664                     nextToken();
 665                     return parseExpressionStatement();
 666                 case BRACES:
 667                 case SEMI:
 668                     nextToken();
 669                     return Completeness.COMPLETE;
 670                 case UNMATCHED:
 671                     nextToken();
 672                     return Completeness.DEFINITELY_INCOMPLETE;
 673                 case EOF:
 674                     switch (in.prevCT.kind) {
 675                         case BRACES:
 676                         case SEMI:
 677                             return Completeness.COMPLETE;
 678                         case IDENTIFIER:
 679                             return isBracesNeeded
 680                                     ? Completeness.DEFINITELY_INCOMPLETE
 681                                     : Completeness.COMPLETE_WITH_SEMI;
 682                         case BRACKETS:
 683                             return Completeness.COMPLETE_WITH_SEMI;
 684                         case DOTSTAR:
 685                             if (isImport) {
 686                                 return Completeness.COMPLETE_WITH_SEMI;
 687                             } else {
 688                                 return Completeness.UNKNOWN;
 689                             }
 690                         default:
 691                             return Completeness.DEFINITELY_INCOMPLETE;
 692                     }
 693                 default:
 694                     return error();
 695             }
 696         }
 697 
 698         public Completeness disambiguateStatementVsExpression() {
 699             if (token.kind == SWITCH) {
 700                 nextToken();
 701                 switch (token.kind) {
 702                     case PARENS:
 703                         nextToken();
 704                         break;
 705                     case UNMATCHED:
 706                         nextToken();
 707                         return Completeness.DEFINITELY_INCOMPLETE;
 708                     case EOF:
 709                         return Completeness.DEFINITELY_INCOMPLETE;
 710                     default:
 711                         return error();
 712                 }
 713                 switch (token.kind) {
 714                     case BRACES:
 715                         nextToken();
 716                         break;
 717                     case UNMATCHED:
 718                         nextToken();
 719                         return Completeness.DEFINITELY_INCOMPLETE;
 720                     case EOF:
 721                         return Completeness.DEFINITELY_INCOMPLETE;
 722                     default:
 723                         return error();
 724                 }
 725                 if (token.kind.valueOp) {
 726                     return parseExpressionOptionalSemi();
 727                 } else {
 728                     return Completeness.COMPLETE;
 729                 }
 730             } else {
 731                 throw new InternalError("Unexpected statement/expression not covered " + token.kind.belongs + " in " + token.kind);
 732             }
 733         }
 734 
 735 
 736         public Completeness disambiguateDeclarationVsExpression() {
 737             // String folding messes up position information.
 738             return parseFactory.apply(pt -> {
 739                 List<? extends Tree> units = pt.units();
 740                 if (units.isEmpty()) {
 741                     return error();
 742                 }
 743                 Tree unitTree = units.get(0);
 744                 switch (unitTree.getKind()) {
 745                     case EXPRESSION_STATEMENT:
 746                         return parseExpressionOptionalSemi();
 747                     case LABELED_STATEMENT:
 748                         if (shouldAbort(IDENTIFIER))  return checkResult;
 749                         if (shouldAbort(COLON))  return checkResult;
 750                         return parseStatement();
 751                     case VARIABLE:
 752                     case IMPORT:
 753                     case CLASS:
 754                     case ENUM:
 755                     case ANNOTATION_TYPE:
 756                     case INTERFACE:
 757                     case METHOD:
 758                         return parseDeclaration();
 759                     default:
 760                         return error();
 761                 }
 762             });
 763         }
 764 
 765         public Completeness parseExpressionStatement() {
 766             if (shouldAbort(parseExpression()))  return checkResult;
 767             return lastly(SEMI);
 768         }
 769 
 770         public Completeness parseExpressionOptionalSemi() {
 771             if (shouldAbort(parseExpression())) return checkResult;
 772             return optionalFinalSemi();
 773         }
 774 
 775         public Completeness parseExpression() {
 776             while (token.kind.isExpression())
 777                 nextToken();
 778             return Completeness.COMPLETE;
 779         }
 780 
 781         public Completeness parseStatement() {
 782             Completeness stat = parseSimpleStatement();
 783             if (stat == null) {
 784                 return parseExpressionStatement();
 785             }
 786             return stat;
 787         }
 788 
 789         /**
 790          * Statement = Block | IF ParExpression Statement [ELSE Statement] | FOR
 791          * "(" ForInitOpt ";" [Expression] ";" ForUpdateOpt ")" Statement | FOR
 792          * "(" FormalParameter : Expression ")" Statement | WHILE ParExpression
 793          * Statement | DO Statement WHILE ParExpression ";" | TRY Block (
 794          * Catches | [Catches] FinallyPart ) | TRY "(" ResourceSpecification
 795          * ";"opt ")" Block [Catches] [FinallyPart] | SWITCH ParExpression "{"
 796          * SwitchBlockStatementGroups "}" | SYNCHRONIZED ParExpression Block |
 797          * RETURN [Expression] ";" | THROW Expression ";" | BREAK [Ident] ";" |
 798          * CONTINUE [Ident] ";" | ASSERT Expression [ ":" Expression ] ";" | ";"
 799          */
 800         public Completeness parseSimpleStatement() {
 801             switch (token.kind) {
 802                 case BRACES:
 803                     return lastly(BRACES);
 804                 case IF: {
 805                     nextToken();
 806                     if (shouldAbort(PARENS))  return checkResult;
 807                     Completeness thenpart = parseStatement();
 808                     if (shouldAbort(thenpart)) return thenpart;
 809                     if (token.kind == ELSE) {
 810                         nextToken();
 811                         return parseStatement();
 812                     }
 813                     return thenpart;
 814 
 815                 }
 816                 case FOR: {
 817                     nextToken();
 818                     if (shouldAbort(PARENS))  return checkResult;
 819                     if (shouldAbort(parseStatement()))  return checkResult;
 820                     return Completeness.COMPLETE;
 821                 }
 822                 case WHILE: {
 823                     nextToken();
 824                     if (shouldAbort(PARENS))  return error();
 825                     return parseStatement();
 826                 }
 827                 case DO: {
 828                     nextToken();
 829                     switch (parseStatement()) {
 830                         case DEFINITELY_INCOMPLETE:
 831                         case CONSIDERED_INCOMPLETE:
 832                         case COMPLETE_WITH_SEMI:
 833                             return Completeness.DEFINITELY_INCOMPLETE;
 834                         case UNKNOWN:
 835                             return error();
 836                         case COMPLETE:
 837                             break;
 838                     }
 839                     if (shouldAbort(WHILE))  return checkResult;
 840                     if (shouldAbort(PARENS)) return checkResult;
 841                     return lastly(SEMI);
 842                 }
 843                 case TRY: {
 844                     boolean hasResources = false;
 845                     nextToken();
 846                     if (token.kind == PARENS) {
 847                         nextToken();
 848                         hasResources = true;
 849                     }
 850                     if (shouldAbort(BRACES))  return checkResult;
 851                     if (token.kind == CATCH || token.kind == FINALLY) {
 852                         while (token.kind == CATCH) {
 853                             if (shouldAbort(CATCH))  return checkResult;
 854                             if (shouldAbort(PARENS)) return checkResult;
 855                             if (shouldAbort(BRACES)) return checkResult;
 856                         }
 857                         if (token.kind == FINALLY) {
 858                             if (shouldAbort(FINALLY))  return checkResult;
 859                             if (shouldAbort(BRACES)) return checkResult;
 860                         }
 861                     } else if (!hasResources) {
 862                         if (token.kind == EOF) {
 863                             return Completeness.DEFINITELY_INCOMPLETE;
 864                         } else {
 865                             return error();
 866                         }
 867                     }
 868                     return Completeness.COMPLETE;
 869                 }
 870                 case SWITCH: {
 871                     nextToken();
 872                     if (shouldAbort(PARENS))  return checkResult;
 873                     return lastly(BRACES);
 874                 }
 875                 case SYNCHRONIZED: {
 876                     nextToken();
 877                     if (shouldAbort(PARENS))  return checkResult;
 878                     return lastly(BRACES);
 879                 }
 880                 case THROW: {
 881                     nextToken();
 882                     if (shouldAbort(parseExpression()))  return checkResult;
 883                     return lastly(SEMI);
 884                 }
 885                 case SEMI:
 886                     return lastly(SEMI);
 887                 case ASSERT:
 888                     nextToken();
 889                     // Crude expression parsing just happily eats the optional colon
 890                     return parseExpressionStatement();
 891                 case RETURN:
 892                 case BREAK:
 893                 case CONTINUE:
 894                     nextToken();
 895                     return parseExpressionStatement();
 896                 // What are these doing here?
 897                 case ELSE:
 898                 case FINALLY:
 899                 case CATCH:
 900                     return error();
 901                 case EOF:
 902                     return Completeness.CONSIDERED_INCOMPLETE;
 903                 default:
 904                     return null;
 905             }
 906         }
 907     }
 908 }