1 /*
   2  * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package jdk.jshell;
  27 
  28 import com.sun.tools.javac.code.Source;
  29 import com.sun.tools.javac.parser.Scanner;
  30 import com.sun.tools.javac.parser.ScannerFactory;
  31 import com.sun.tools.javac.parser.Tokens.Token;
  32 import com.sun.tools.javac.parser.Tokens.TokenKind;
  33 import com.sun.tools.javac.util.Context;
  34 import com.sun.tools.javac.util.DiagnosticSource;
  35 import com.sun.tools.javac.util.JCDiagnostic;
  36 import com.sun.tools.javac.util.JCDiagnostic.DiagnosticFlag;
  37 import com.sun.tools.javac.util.JCDiagnostic.DiagnosticPosition;
  38 import com.sun.tools.javac.util.JCDiagnostic.Error;
  39 import com.sun.tools.javac.util.Log;
  40 import java.io.PrintWriter;
  41 import java.io.StringWriter;
  42 import java.util.ArrayDeque;
  43 import java.util.Deque;
  44 import java.util.EnumMap;
  45 import java.util.Iterator;
  46 import jdk.jshell.SourceCodeAnalysis.Completeness;
  47 import com.sun.source.tree.Tree;
  48 import static jdk.jshell.CompletenessAnalyzer.TK.*;
  49 import jdk.jshell.TaskFactory.ParseTask;
  50 import jdk.jshell.TaskFactory.Worker;
  51 import java.util.List;
  52 import java.util.function.Function;
  53 import java.util.function.Supplier;
  54 
  55 /**
  56  * Low level scanner to determine completeness of input.
  57  * @author Robert Field
  58  */
  59 class CompletenessAnalyzer {
  60 
  61     private final ScannerFactory scannerFactory;
  62     private final JShell proc;
  63 
  64     private static Completeness error() {
  65         return Completeness.UNKNOWN;  // For breakpointing
  66     }
  67 
  68     static class CaInfo {
  69 
  70         CaInfo(Completeness status, int unitEndPos) {
  71             this.status = status;
  72             this.unitEndPos = unitEndPos;
  73         }
  74         final int unitEndPos;
  75         final Completeness status;
  76     }
  77 
  78     CompletenessAnalyzer(JShell proc) {
  79         this.proc = proc;
  80         Context context = new Context();
  81         Log log = CaLog.createLog(context);
  82         context.put(Log.class, log);
  83         context.put(Source.class, Source.JDK9);
  84         scannerFactory = ScannerFactory.instance(context);
  85     }
  86 
  87     CaInfo scan(String s) {
  88         try {
  89             Parser parser = new Parser(
  90                     () -> new Matched(scannerFactory.newScanner(s, false)),
  91                     worker -> proc.taskFactory.parse(s, worker));
  92             Completeness stat = parser.parseUnit();
  93             int endPos = stat == Completeness.UNKNOWN
  94                     ? s.length()
  95                     : parser.endPos();
  96             return new CaInfo(stat, endPos);
  97         } catch (SyntaxException ex) {
  98             return new CaInfo(error(), s.length());
  99         }
 100     }
 101 
 102     @SuppressWarnings("serial")             // serialVersionUID intentionally omitted
 103     private static class SyntaxException extends RuntimeException {
 104     }
 105 
 106     private static void die() {
 107         throw new SyntaxException();
 108     }
 109 
 110     /**
 111      * Subclass of Log used by compiler API to die on error and ignore
 112      * other messages
 113      */
 114     private static class CaLog extends Log {
 115 
 116         private static CaLog createLog(Context context) {
 117             PrintWriter pw = new PrintWriter(new StringWriter());
 118             CaLog log = new CaLog(context, pw);
 119             context.put(logKey, log);
 120             return log;
 121         }
 122 
 123         private CaLog(Context context, PrintWriter pw) {
 124             super(context, pw);
 125             this.source = DiagnosticSource.NO_SOURCE;
 126         }
 127 
 128         @Override
 129         public void error(String key, Object... args) {
 130             die();
 131         }
 132 
 133         @Override
 134         public void error(int pos, Error errorKey) {
 135             die();
 136         }
 137 
 138         @Override
 139         public void error(int pos, String key, Object... args) {
 140             die();
 141         }
 142 
 143         @Override
 144         public void report(JCDiagnostic diagnostic) {
 145             // Ignore
 146         }
 147     }
 148 
 149     // Location position kinds -- a token is ...
 150     private static final int XEXPR         = 0b1;                       // OK in expression (not first)
 151     private static final int XDECL         = 0b10;                      // OK in declaration (not first)
 152     private static final int XSTMT         = 0b100;                     // OK in statement framework (not first)
 153     private static final int XEXPR1o       = 0b1000;                    // OK first in expression
 154     private static final int XDECL1o       = 0b10000;                   // OK first in declaration
 155     private static final int XSTMT1o       = 0b100000;                  // OK first or only in statement framework
 156     private static final int XEXPR1        = XEXPR1o | XEXPR;           // OK in expression (anywhere)
 157     private static final int XDECL1        = XDECL1o | XDECL;           // OK in declaration (anywhere)
 158     private static final int XSTMT1        = XSTMT1o | XSTMT;           // OK in statement framework (anywhere)
 159     private static final int XANY1         = XEXPR1o | XDECL1o | XSTMT1o;  // Mask: first in statement, declaration, or expression
 160     private static final int XTERM         = 0b100000000;               // Can terminate (last before EOF)
 161     private static final int XSTART        = 0b1000000000;              // Boundary, must be XTERM before
 162     private static final int XERRO         = 0b10000000000;             // Is an error
 163     private static final int XBRACESNEEDED = 0b100000000000;            // Expect {ANY} LBRACE
 164 
 165     /**
 166      * An extension of the compiler's TokenKind which adds our combined/processed
 167      * kinds. Also associates each TK with a union of acceptable kinds of code
 168      * position it can occupy.  For example: IDENTIFER is XEXPR1|XDECL1|XTERM,
 169      * meaning it can occur in expressions or declarations (but not in the
 170      * framework of a statement and that can be the final (terminating) token
 171      * in a snippet.
 172      * <P>
 173      * There must be a TK defined for each compiler TokenKind, an exception
 174      * will
 175      * be thrown if a TokenKind is defined and a corresponding TK is not. Add a
 176      * new TK in the appropriate category. If it is like an existing category
 177      * (e.g. a new modifier or type this may be all that is needed.  If it
 178      * is bracketing or modifies the acceptable positions of other tokens,
 179      * please closely examine the needed changes to this scanner.
 180      */
 181     static enum TK {
 182 
 183         // Special
 184         EOF(TokenKind.EOF, 0),  //
 185         ERROR(TokenKind.ERROR, XERRO),  //
 186         IDENTIFIER(TokenKind.IDENTIFIER, XEXPR1|XDECL1|XTERM),  //
 187         UNDERSCORE(TokenKind.UNDERSCORE, XERRO),  //  _
 188         CLASS(TokenKind.CLASS, XEXPR|XDECL1|XBRACESNEEDED),  //  class decl (MAPPED: DOTCLASS)
 189         MONKEYS_AT(TokenKind.MONKEYS_AT, XEXPR|XDECL1),  //  @
 190         IMPORT(TokenKind.IMPORT, XDECL1|XSTART),  //  import -- consider declaration
 191         SEMI(TokenKind.SEMI, XSTMT1|XTERM|XSTART),  //  ;
 192 
 193         // Shouldn't see -- error
 194         PACKAGE(TokenKind.PACKAGE, XERRO),  //  package
 195         CONST(TokenKind.CONST, XERRO),  //  reserved keyword -- const
 196         GOTO(TokenKind.GOTO, XERRO),  //  reserved keyword -- goto
 197         CUSTOM(TokenKind.CUSTOM, XERRO),  // No uses
 198 
 199         // Declarations
 200         ENUM(TokenKind.ENUM, XDECL1|XBRACESNEEDED),  //  enum
 201         IMPLEMENTS(TokenKind.IMPLEMENTS, XDECL),  //  implements
 202         INTERFACE(TokenKind.INTERFACE, XDECL1|XBRACESNEEDED),  //  interface
 203         THROWS(TokenKind.THROWS, XDECL|XBRACESNEEDED),  //  throws
 204 
 205         // Primarive type names
 206         BOOLEAN(TokenKind.BOOLEAN, XEXPR1|XDECL1),  //  boolean
 207         BYTE(TokenKind.BYTE, XEXPR1|XDECL1),  //  byte
 208         CHAR(TokenKind.CHAR, XEXPR1|XDECL1),  //  char
 209         DOUBLE(TokenKind.DOUBLE, XEXPR1|XDECL1),  //  double
 210         FLOAT(TokenKind.FLOAT, XEXPR1|XDECL1),  //  float
 211         INT(TokenKind.INT, XEXPR1|XDECL1),  //  int
 212         LONG(TokenKind.LONG, XEXPR1|XDECL1),  //  long
 213         SHORT(TokenKind.SHORT, XEXPR1|XDECL1),  //  short
 214         VOID(TokenKind.VOID, XEXPR1|XDECL1),  //  void
 215 
 216         // Modifiers keywords
 217         ABSTRACT(TokenKind.ABSTRACT, XDECL1),  //  abstract
 218         FINAL(TokenKind.FINAL, XDECL1),  //  final
 219         NATIVE(TokenKind.NATIVE, XDECL1),  //  native
 220         STATIC(TokenKind.STATIC, XDECL1),  //  static
 221         STRICTFP(TokenKind.STRICTFP, XDECL1),  //  strictfp
 222         PRIVATE(TokenKind.PRIVATE, XDECL1),  //  private
 223         PROTECTED(TokenKind.PROTECTED, XDECL1),  //  protected
 224         PUBLIC(TokenKind.PUBLIC, XDECL1),  //  public
 225         TRANSIENT(TokenKind.TRANSIENT, XDECL1),  //  transient
 226         VOLATILE(TokenKind.VOLATILE, XDECL1),  //  volatile
 227 
 228         // Declarations and type parameters (thus expressions)
 229         EXTENDS(TokenKind.EXTENDS, XEXPR|XDECL),  //  extends
 230         COMMA(TokenKind.COMMA, XEXPR|XDECL),  //  ,
 231         AMP(TokenKind.AMP, XEXPR|XDECL, true),  //  &
 232         GT(TokenKind.GT, XEXPR|XDECL, true),  //  >
 233         LT(TokenKind.LT, XEXPR|XDECL1, true),  //  <
 234         LTLT(TokenKind.LTLT, XEXPR|XDECL1, true),  //  <<
 235         GTGT(TokenKind.GTGT, XEXPR|XDECL, true),  //  >>
 236         GTGTGT(TokenKind.GTGTGT, XEXPR|XDECL, true),  //  >>>
 237         QUES(TokenKind.QUES, XEXPR|XDECL, true),  //  ?
 238         DOT(TokenKind.DOT, XEXPR|XDECL),  //  .
 239         STAR(TokenKind.STAR, XEXPR, true),  //  * (MAPPED: DOTSTAR)
 240 
 241         // Statement keywords
 242         ASSERT(TokenKind.ASSERT, XSTMT1|XSTART),  //  assert
 243         BREAK(TokenKind.BREAK, XSTMT1|XTERM|XSTART),  //  break
 244         CATCH(TokenKind.CATCH, XSTMT1|XSTART),  //  catch
 245         CONTINUE(TokenKind.CONTINUE, XSTMT1|XTERM|XSTART),  //  continue
 246         DO(TokenKind.DO, XSTMT1|XSTART),  //  do
 247         ELSE(TokenKind.ELSE, XSTMT1|XTERM|XSTART),  //  else
 248         FINALLY(TokenKind.FINALLY, XSTMT1|XSTART),  //  finally
 249         FOR(TokenKind.FOR, XSTMT1|XSTART),  //  for
 250         IF(TokenKind.IF, XSTMT1|XSTART),  //  if
 251         RETURN(TokenKind.RETURN, XSTMT1|XTERM|XSTART),  //  return
 252         SWITCH(TokenKind.SWITCH, XSTMT1|XEXPR1),  //  switch
 253         SYNCHRONIZED(TokenKind.SYNCHRONIZED, XSTMT1|XDECL),  //  synchronized
 254         THROW(TokenKind.THROW, XSTMT1|XSTART),  //  throw
 255         TRY(TokenKind.TRY, XSTMT1|XSTART),  //  try
 256         WHILE(TokenKind.WHILE, XSTMT1|XSTART),  //  while
 257 
 258         // Statement keywords that we shouldn't see -- inside braces
 259         CASE(TokenKind.CASE, XSTMT|XSTART),  //  case
 260         DEFAULT(TokenKind.DEFAULT, XSTMT|XSTART),  //  default method, default case -- neither we should see
 261 
 262         // Expressions (can terminate)
 263         INTLITERAL(TokenKind.INTLITERAL, XEXPR1|XTERM),  //
 264         LONGLITERAL(TokenKind.LONGLITERAL, XEXPR1|XTERM),  //
 265         FLOATLITERAL(TokenKind.FLOATLITERAL, XEXPR1|XTERM),  //
 266         DOUBLELITERAL(TokenKind.DOUBLELITERAL, XEXPR1|XTERM),  //
 267         CHARLITERAL(TokenKind.CHARLITERAL, XEXPR1|XTERM),  //
 268         STRINGLITERAL(TokenKind.STRINGLITERAL, XEXPR1|XTERM),  //
 269         TRUE(TokenKind.TRUE, XEXPR1|XTERM),  //  true
 270         FALSE(TokenKind.FALSE, XEXPR1|XTERM),  //  false
 271         NULL(TokenKind.NULL, XEXPR1|XTERM),  //  null
 272         THIS(TokenKind.THIS, XEXPR1|XTERM),  //  this  -- shouldn't see
 273 
 274         // Expressions maybe terminate  //TODO handle these case separately
 275         PLUSPLUS(TokenKind.PLUSPLUS, XEXPR1|XTERM),  //  ++
 276         SUBSUB(TokenKind.SUBSUB, XEXPR1|XTERM),  //  --
 277 
 278         // Expressions cannot terminate
 279         INSTANCEOF(TokenKind.INSTANCEOF, XEXPR, true),  //  instanceof
 280         NEW(TokenKind.NEW, XEXPR1),  //  new (MAPPED: COLCOLNEW)
 281         SUPER(TokenKind.SUPER, XEXPR1|XDECL),  //  super -- shouldn't see as rec. But in type parameters
 282         ARROW(TokenKind.ARROW, XEXPR),  //  ->
 283         COLCOL(TokenKind.COLCOL, XEXPR),  //  ::
 284         LPAREN(TokenKind.LPAREN, XEXPR),  //  (
 285         RPAREN(TokenKind.RPAREN, XEXPR),  //  )
 286         LBRACE(TokenKind.LBRACE, XEXPR),  //  {
 287         RBRACE(TokenKind.RBRACE, XEXPR),  //  }
 288         LBRACKET(TokenKind.LBRACKET, XEXPR),  //  [
 289         RBRACKET(TokenKind.RBRACKET, XEXPR),  //  ]
 290         ELLIPSIS(TokenKind.ELLIPSIS, XEXPR),  //  ...
 291         EQ(TokenKind.EQ, XEXPR),  //  =
 292         BANG(TokenKind.BANG, XEXPR1),  //  !
 293         TILDE(TokenKind.TILDE, XEXPR1),  //  ~
 294         COLON(TokenKind.COLON, XEXPR|XTERM),  //  :
 295         EQEQ(TokenKind.EQEQ, XEXPR, true),  //  ==
 296         LTEQ(TokenKind.LTEQ, XEXPR, true),  //  <=
 297         GTEQ(TokenKind.GTEQ, XEXPR, true),  //  >=
 298         BANGEQ(TokenKind.BANGEQ, XEXPR, true),  //  !=
 299         AMPAMP(TokenKind.AMPAMP, XEXPR, true),  //  &&
 300         BARBAR(TokenKind.BARBAR, XEXPR, true),  //  ||
 301         PLUS(TokenKind.PLUS, XEXPR1, true),  //  +
 302         SUB(TokenKind.SUB, XEXPR1, true),  //  -
 303         SLASH(TokenKind.SLASH, XEXPR, true),  //  /
 304         BAR(TokenKind.BAR, XEXPR, true),  //  |
 305         CARET(TokenKind.CARET, XEXPR, true),  //  ^
 306         PERCENT(TokenKind.PERCENT, XEXPR, true),  //  %
 307         PLUSEQ(TokenKind.PLUSEQ, XEXPR),  //  +=
 308         SUBEQ(TokenKind.SUBEQ, XEXPR),  //  -=
 309         STAREQ(TokenKind.STAREQ, XEXPR),  //  *=
 310         SLASHEQ(TokenKind.SLASHEQ, XEXPR),  //  /=
 311         AMPEQ(TokenKind.AMPEQ, XEXPR),  //  &=
 312         BAREQ(TokenKind.BAREQ, XEXPR),  //  |=
 313         CARETEQ(TokenKind.CARETEQ, XEXPR),  //  ^=
 314         PERCENTEQ(TokenKind.PERCENTEQ, XEXPR),  //  %=
 315         LTLTEQ(TokenKind.LTLTEQ, XEXPR),  //  <<=
 316         GTGTEQ(TokenKind.GTGTEQ, XEXPR),  //  >>=
 317         GTGTGTEQ(TokenKind.GTGTGTEQ, XEXPR),  //  >>>=
 318 
 319         // combined/processed kinds
 320         UNMATCHED(XERRO),
 321         PARENS(XEXPR1|XDECL|XSTMT|XTERM),
 322         BRACKETS(XEXPR|XDECL|XTERM),
 323         BRACES(XSTMT1|XEXPR|XTERM),
 324         DOTSTAR(XDECL|XTERM),  // import foo.*
 325         COLCOLNEW(XEXPR|XTERM),  //  :: new
 326         DOTCLASS(XEXPR|XTERM),  //  class decl and .class
 327         ;
 328 
 329         static final EnumMap<TokenKind,TK> tokenKindToTKMap = new EnumMap<>(TokenKind.class);
 330 
 331         final TokenKind tokenKind;
 332         final int belongs;
 333         final boolean valueOp;
 334         Function<TK,TK> mapping;
 335 
 336         TK(int b) {
 337             this(null, b);
 338         }
 339 
 340         TK(TokenKind tokenKind, int b) {
 341             this(tokenKind, b, false);
 342         }
 343 
 344         TK(TokenKind tokenKind, int b, boolean valueOp) {
 345             this.tokenKind = tokenKind;
 346             this.belongs = b;
 347             this.valueOp = valueOp;
 348             this.mapping = null;
 349         }
 350 
 351         private static TK tokenKindToTK(TK prev, TokenKind kind) {
 352             TK tk = tokenKindToTKMap.get(kind);
 353             if (tk == null) {
 354                 System.err.printf("No corresponding %s for %s: %s\n",
 355                         TK.class.getCanonicalName(),
 356                         TokenKind.class.getCanonicalName(),
 357                         kind);
 358                 throw new InternalError("No corresponding TK for TokenKind: " + kind);
 359             }
 360             return tk.mapping != null
 361                     ? tk.mapping.apply(prev)
 362                     : tk;
 363         }
 364 
 365         boolean isOkToTerminate() {
 366             return (belongs & XTERM) != 0;
 367         }
 368 
 369         boolean isExpression() {
 370             return (belongs & XEXPR) != 0;
 371         }
 372 
 373         boolean isDeclaration() {
 374             return (belongs & XDECL) != 0;
 375         }
 376 
 377         boolean isError() {
 378             return (belongs & XERRO) != 0;
 379         }
 380 
 381         boolean isStart() {
 382             return (belongs & XSTART) != 0;
 383         }
 384 
 385         boolean isBracesNeeded() {
 386             return (belongs & XBRACESNEEDED) != 0;
 387         }
 388 
 389         /**
 390          * After construction, check that all compiler TokenKind values have
 391          * corresponding TK values.
 392          */
 393         static {
 394             for (TK tk : TK.values()) {
 395                 if (tk.tokenKind != null) {
 396                     tokenKindToTKMap.put(tk.tokenKind, tk);
 397                 }
 398             }
 399             for (TokenKind kind : TokenKind.values()) {
 400                 tokenKindToTK(null, kind); // assure they can be retrieved without error
 401             }
 402             // Mappings of disambiguated contexts
 403             STAR.mapping  = prev -> prev == DOT ? DOTSTAR : STAR;
 404             NEW.mapping   = prev -> prev == COLCOL ? COLCOLNEW : NEW;
 405             CLASS.mapping = prev -> prev == DOT ? DOTCLASS : CLASS;
 406         }
 407     }
 408 
 409     /**
 410      * A completeness scanner token.
 411      */
 412     private static class CT {
 413 
 414         /** The token kind */
 415         public final TK kind;
 416 
 417         /** The end position of this token */
 418         public final int endPos;
 419 
 420         /** The error message **/
 421         public final String message;
 422 
 423         private CT(TK tk, Token tok, String msg) {
 424             this.kind = tk;
 425             this.endPos = tok.endPos;
 426             this.message = msg;
 427             //throw new InternalError(msg); /* for debugging */
 428         }
 429 
 430         private CT(TK tk, Token tok) {
 431             this.kind = tk;
 432             this.endPos = tok.endPos;
 433             this.message = null;
 434         }
 435 
 436         private CT(TK tk, int endPos) {
 437             this.kind = tk;
 438             this.endPos = endPos;
 439             this.message = null;
 440         }
 441     }
 442 
 443     /**
 444      * Look for matching tokens (like parens) and other special cases, like "new"
 445      */
 446     private static class Matched implements Iterator<CT> {
 447 
 448         private final Scanner scanner;
 449         private Token current;
 450         private CT prevCT;
 451         private CT currentCT;
 452         private final Deque<Token> stack = new ArrayDeque<>();
 453 
 454         Matched(Scanner scanner) {
 455             this.scanner = scanner;
 456             advance();
 457             prevCT = currentCT = new CT(SEMI, 0); // So is valid for testing
 458         }
 459 
 460         @Override
 461         public boolean hasNext() {
 462             return currentCT.kind != EOF;
 463         }
 464 
 465         private Token advance() {
 466             Token prev = current;
 467             scanner.nextToken();
 468             current = scanner.token();
 469             return prev;
 470         }
 471 
 472         @Override
 473         public CT next() {
 474             prevCT = currentCT;
 475             currentCT = nextCT();
 476             return currentCT;
 477         }
 478 
 479         private CT match(TK tk, TokenKind open) {
 480             Token tok = advance();
 481             db("match desired-tk=%s, open=%s, seen-tok=%s", tk, open, tok.kind);
 482             if (stack.isEmpty()) {
 483                 return new CT(ERROR, tok, "Encountered '" + tok + "' with no opening '" + open + "'");
 484             }
 485             Token p = stack.pop();
 486             if (p.kind != open) {
 487                 return new CT(ERROR, tok, "No match for '" + p + "' instead encountered '" + tok + "'");
 488             }
 489             return new CT(tk, tok);
 490         }
 491 
 492         private void db(String format, Object ... args) {
 493 //            System.err.printf(format, args);
 494 //            System.err.printf(" -- stack(");
 495 //            if (stack.isEmpty()) {
 496 //
 497 //            } else {
 498 //                for (Token tok : stack) {
 499 //                    System.err.printf("%s ", tok.kind);
 500 //                }
 501 //            }
 502 //            System.err.printf(") current=%s / currentCT=%s\n", current.kind, currentCT.kind);
 503         }
 504 
 505         /**
 506          * @return the next scanner token
 507          */
 508         private CT nextCT() {
 509             // TODO Annotations?
 510             TK prevTK = currentCT.kind;
 511             while (true) {
 512                 db("nextCT");
 513                 CT ct;
 514                 switch (current.kind) {
 515                     case EOF:
 516                         db("eof");
 517                         if (stack.isEmpty()) {
 518                             ct = new CT(EOF, current);
 519                         } else {
 520                             TokenKind unmatched = stack.pop().kind;
 521                             stack.clear(); // So we will get EOF next time
 522                             ct = new CT(UNMATCHED, current, "Unmatched " + unmatched);
 523                         }
 524                         break;
 525                     case LPAREN:
 526                     case LBRACE:
 527                     case LBRACKET:
 528                         stack.push(advance());
 529                         prevTK = SEMI; // new start
 530                         continue;
 531                     case RPAREN:
 532                         ct = match(PARENS, TokenKind.LPAREN);
 533                         break;
 534                     case RBRACE:
 535                         ct = match(BRACES, TokenKind.LBRACE);
 536                         break;
 537                     case RBRACKET:
 538                         ct = match(BRACKETS, TokenKind.LBRACKET);
 539                         break;
 540                     default:
 541                         ct = new CT(TK.tokenKindToTK(prevTK, current.kind), advance());
 542                         break;
 543                 }
 544                 // Detect an error if we are at starting position and the last
 545                 // token wasn't a terminating one.  Special case: within braces,
 546                 // comma can proceed semicolon, e.g. the values list in enum
 547                 if (ct.kind.isStart() && !prevTK.isOkToTerminate() && prevTK != COMMA) {
 548                     return new CT(ERROR, current, "No '" + prevTK + "' before '" + ct.kind + "'");
 549                 }
 550                 if (stack.isEmpty() || ct.kind.isError()) {
 551                     return ct;
 552                 }
 553                 prevTK = ct.kind;
 554             }
 555         }
 556     }
 557 
 558     /**
 559      * Fuzzy parser based on token kinds
 560      */
 561     private static class Parser {
 562 
 563         private final Supplier<Matched> matchedFactory;
 564         private final Function<Worker<ParseTask, Completeness>, Completeness> parseFactory;
 565         private Matched in;
 566         private CT token;
 567         private Completeness checkResult;
 568 
 569         Parser(Supplier<Matched> matchedFactory,
 570                Function<Worker<ParseTask, Completeness>, Completeness> parseFactory) {
 571             this.matchedFactory = matchedFactory;
 572             this.parseFactory = parseFactory;
 573             resetInput();
 574         }
 575 
 576         final void resetInput() {
 577             this.in = matchedFactory.get();
 578             nextToken();
 579         }
 580 
 581         final void nextToken() {
 582             in.next();
 583             token = in.currentCT;
 584         }
 585 
 586         boolean shouldAbort(TK tk) {
 587             if (token.kind == tk) {
 588                 nextToken();
 589                 return false;
 590             }
 591             switch (token.kind) {
 592                 case EOF:
 593                     checkResult = ((tk == SEMI) && in.prevCT.kind.isOkToTerminate())
 594                             ? Completeness.COMPLETE_WITH_SEMI
 595                             : Completeness.DEFINITELY_INCOMPLETE;
 596                     return true;
 597                 case UNMATCHED:
 598                     checkResult = Completeness.DEFINITELY_INCOMPLETE;
 599                     return true;
 600                 default:
 601                     checkResult = error();
 602                     return true;
 603 
 604             }
 605         }
 606 
 607         Completeness lastly(TK tk) {
 608             if (shouldAbort(tk))  return checkResult;
 609             return Completeness.COMPLETE;
 610         }
 611 
 612         Completeness optionalFinalSemi() {
 613             if (!shouldAbort(SEMI)) return Completeness.COMPLETE;
 614             if (checkResult == Completeness.COMPLETE_WITH_SEMI) return Completeness.COMPLETE;
 615             return checkResult;
 616         }
 617 
 618         boolean shouldAbort(Completeness flags) {
 619             checkResult = flags;
 620             return flags != Completeness.COMPLETE;
 621         }
 622 
 623         public int endPos() {
 624             return in.prevCT.endPos;
 625         }
 626 
 627         public Completeness parseUnit() {
 628             //System.err.printf("%s:  belongs %o  XANY1 %o\n", token.kind, token.kind.belongs, token.kind.belongs & XANY1);
 629             switch (token.kind.belongs & XANY1) {
 630                 case XEXPR1o:
 631                     return parseExpressionOptionalSemi();
 632                 case XSTMT1o: {
 633                     Completeness stat = parseSimpleStatement();
 634                     return stat==null? error() : stat;
 635                 }
 636                 case XDECL1o:
 637                     return parseDeclaration();
 638                 case XSTMT1o | XDECL1o:
 639                 case XEXPR1o | XDECL1o:
 640                     return disambiguateDeclarationVsExpression();
 641                 case 0:
 642                     if ((token.kind.belongs & XERRO) != 0) {
 643                         return parseExpressionStatement(); // Let this gen the status
 644                     }
 645                     return error();
 646                 case XSTMT1o | XEXPR1o:
 647                     return disambiguateStatementVsExpression();
 648                 default:
 649                     throw new InternalError("Case not covered " + token.kind.belongs + " in " + token.kind);
 650             }
 651         }
 652 
 653         public Completeness parseDeclaration() {
 654             boolean isImport = token.kind == IMPORT;
 655             boolean isBracesNeeded = false;
 656             while (token.kind.isDeclaration()) {
 657                 isBracesNeeded |= token.kind.isBracesNeeded();
 658                 nextToken();
 659             }
 660             switch (token.kind) {
 661                 case EQ:
 662                     nextToken();
 663                     return parseExpressionStatement();
 664                 case BRACES:
 665                 case SEMI:
 666                     nextToken();
 667                     return Completeness.COMPLETE;
 668                 case UNMATCHED:
 669                     nextToken();
 670                     return Completeness.DEFINITELY_INCOMPLETE;
 671                 case EOF:
 672                     switch (in.prevCT.kind) {
 673                         case BRACES:
 674                         case SEMI:
 675                             return Completeness.COMPLETE;
 676                         case IDENTIFIER:
 677                             return isBracesNeeded
 678                                     ? Completeness.DEFINITELY_INCOMPLETE
 679                                     : Completeness.COMPLETE_WITH_SEMI;
 680                         case BRACKETS:
 681                             return Completeness.COMPLETE_WITH_SEMI;
 682                         case DOTSTAR:
 683                             if (isImport) {
 684                                 return Completeness.COMPLETE_WITH_SEMI;
 685                             } else {
 686                                 return Completeness.UNKNOWN;
 687                             }
 688                         default:
 689                             return Completeness.DEFINITELY_INCOMPLETE;
 690                     }
 691                 default:
 692                     return error();
 693             }
 694         }
 695 
 696         public Completeness disambiguateStatementVsExpression() {
 697             if (token.kind == SWITCH) {
 698                 nextToken();
 699                 switch (token.kind) {
 700                     case PARENS:
 701                         nextToken();
 702                         break;
 703                     case UNMATCHED:
 704                         nextToken();
 705                         return Completeness.DEFINITELY_INCOMPLETE;
 706                     case EOF:
 707                         return Completeness.DEFINITELY_INCOMPLETE;
 708                     default:
 709                         return error();
 710                 }
 711                 switch (token.kind) {
 712                     case BRACES:
 713                         nextToken();
 714                         break;
 715                     case UNMATCHED:
 716                         nextToken();
 717                         return Completeness.DEFINITELY_INCOMPLETE;
 718                     case EOF:
 719                         return Completeness.DEFINITELY_INCOMPLETE;
 720                     default:
 721                         return error();
 722                 }
 723                 if (token.kind.valueOp) {
 724                     return parseExpressionOptionalSemi();
 725                 } else {
 726                     return Completeness.COMPLETE;
 727                 }
 728             } else {
 729                 throw new InternalError("Unexpected statement/expression not covered " + token.kind.belongs + " in " + token.kind);
 730             }
 731         }
 732 
 733 
 734         public Completeness disambiguateDeclarationVsExpression() {
 735             // String folding messes up position information.
 736             return parseFactory.apply(pt -> {
 737                 List<? extends Tree> units = pt.units();
 738                 if (units.isEmpty()) {
 739                     return error();
 740                 }
 741                 Tree unitTree = units.get(0);
 742                 switch (unitTree.getKind()) {
 743                     case EXPRESSION_STATEMENT:
 744                         return parseExpressionOptionalSemi();
 745                     case LABELED_STATEMENT:
 746                         if (shouldAbort(IDENTIFIER))  return checkResult;
 747                         if (shouldAbort(COLON))  return checkResult;
 748                         return parseStatement();
 749                     case VARIABLE:
 750                     case IMPORT:
 751                     case CLASS:
 752                     case ENUM:
 753                     case ANNOTATION_TYPE:
 754                     case INTERFACE:
 755                     case METHOD:
 756                         return parseDeclaration();
 757                     default:
 758                         return error();
 759                 }
 760             });
 761         }
 762 
 763         public Completeness parseExpressionStatement() {
 764             if (shouldAbort(parseExpression()))  return checkResult;
 765             return lastly(SEMI);
 766         }
 767 
 768         public Completeness parseExpressionOptionalSemi() {
 769             if (shouldAbort(parseExpression())) return checkResult;
 770             return optionalFinalSemi();
 771         }
 772 
 773         public Completeness parseExpression() {
 774             while (token.kind.isExpression())
 775                 nextToken();
 776             return Completeness.COMPLETE;
 777         }
 778 
 779         public Completeness parseStatement() {
 780             Completeness stat = parseSimpleStatement();
 781             if (stat == null) {
 782                 return parseExpressionStatement();
 783             }
 784             return stat;
 785         }
 786 
 787         /**
 788          * Statement = Block | IF ParExpression Statement [ELSE Statement] | FOR
 789          * "(" ForInitOpt ";" [Expression] ";" ForUpdateOpt ")" Statement | FOR
 790          * "(" FormalParameter : Expression ")" Statement | WHILE ParExpression
 791          * Statement | DO Statement WHILE ParExpression ";" | TRY Block (
 792          * Catches | [Catches] FinallyPart ) | TRY "(" ResourceSpecification
 793          * ";"opt ")" Block [Catches] [FinallyPart] | SWITCH ParExpression "{"
 794          * SwitchBlockStatementGroups "}" | SYNCHRONIZED ParExpression Block |
 795          * RETURN [Expression] ";" | THROW Expression ";" | BREAK [Ident] ";" |
 796          * CONTINUE [Ident] ";" | ASSERT Expression [ ":" Expression ] ";" | ";"
 797          */
 798         public Completeness parseSimpleStatement() {
 799             switch (token.kind) {
 800                 case BRACES:
 801                     return lastly(BRACES);
 802                 case IF: {
 803                     nextToken();
 804                     if (shouldAbort(PARENS))  return checkResult;
 805                     Completeness thenpart = parseStatement();
 806                     if (shouldAbort(thenpart)) return thenpart;
 807                     if (token.kind == ELSE) {
 808                         nextToken();
 809                         return parseStatement();
 810                     }
 811                     return thenpart;
 812 
 813                 }
 814                 case FOR: {
 815                     nextToken();
 816                     if (shouldAbort(PARENS))  return checkResult;
 817                     if (shouldAbort(parseStatement()))  return checkResult;
 818                     return Completeness.COMPLETE;
 819                 }
 820                 case WHILE: {
 821                     nextToken();
 822                     if (shouldAbort(PARENS))  return error();
 823                     return parseStatement();
 824                 }
 825                 case DO: {
 826                     nextToken();
 827                     switch (parseStatement()) {
 828                         case DEFINITELY_INCOMPLETE:
 829                         case CONSIDERED_INCOMPLETE:
 830                         case COMPLETE_WITH_SEMI:
 831                             return Completeness.DEFINITELY_INCOMPLETE;
 832                         case UNKNOWN:
 833                             return error();
 834                         case COMPLETE:
 835                             break;
 836                     }
 837                     if (shouldAbort(WHILE))  return checkResult;
 838                     if (shouldAbort(PARENS)) return checkResult;
 839                     return lastly(SEMI);
 840                 }
 841                 case TRY: {
 842                     boolean hasResources = false;
 843                     nextToken();
 844                     if (token.kind == PARENS) {
 845                         nextToken();
 846                         hasResources = true;
 847                     }
 848                     if (shouldAbort(BRACES))  return checkResult;
 849                     if (token.kind == CATCH || token.kind == FINALLY) {
 850                         while (token.kind == CATCH) {
 851                             if (shouldAbort(CATCH))  return checkResult;
 852                             if (shouldAbort(PARENS)) return checkResult;
 853                             if (shouldAbort(BRACES)) return checkResult;
 854                         }
 855                         if (token.kind == FINALLY) {
 856                             if (shouldAbort(FINALLY))  return checkResult;
 857                             if (shouldAbort(BRACES)) return checkResult;
 858                         }
 859                     } else if (!hasResources) {
 860                         if (token.kind == EOF) {
 861                             return Completeness.DEFINITELY_INCOMPLETE;
 862                         } else {
 863                             return error();
 864                         }
 865                     }
 866                     return Completeness.COMPLETE;
 867                 }
 868                 case SWITCH: {
 869                     nextToken();
 870                     if (shouldAbort(PARENS))  return checkResult;
 871                     return lastly(BRACES);
 872                 }
 873                 case SYNCHRONIZED: {
 874                     nextToken();
 875                     if (shouldAbort(PARENS))  return checkResult;
 876                     return lastly(BRACES);
 877                 }
 878                 case THROW: {
 879                     nextToken();
 880                     if (shouldAbort(parseExpression()))  return checkResult;
 881                     return lastly(SEMI);
 882                 }
 883                 case SEMI:
 884                     return lastly(SEMI);
 885                 case ASSERT:
 886                     nextToken();
 887                     // Crude expression parsing just happily eats the optional colon
 888                     return parseExpressionStatement();
 889                 case RETURN:
 890                 case BREAK:
 891                 case CONTINUE:
 892                     nextToken();
 893                     return parseExpressionStatement();
 894                 // What are these doing here?
 895                 case ELSE:
 896                 case FINALLY:
 897                 case CATCH:
 898                     return error();
 899                 case EOF:
 900                     return Completeness.CONSIDERED_INCOMPLETE;
 901                 default:
 902                     return null;
 903             }
 904         }
 905     }
 906 }