1 /*
   2  * Copyright (c) 2015, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package jdk.jshell;
  27 
  28 import com.sun.tools.javac.code.Source;
  29 import com.sun.tools.javac.parser.Scanner;
  30 import com.sun.tools.javac.parser.ScannerFactory;
  31 import com.sun.tools.javac.parser.Tokens.Token;
  32 import com.sun.tools.javac.parser.Tokens.TokenKind;
  33 import com.sun.tools.javac.util.Context;
  34 import com.sun.tools.javac.util.DiagnosticSource;
  35 import com.sun.tools.javac.util.JCDiagnostic;
  36 import com.sun.tools.javac.util.JCDiagnostic.DiagnosticFlag;
  37 import com.sun.tools.javac.util.JCDiagnostic.DiagnosticPosition;
  38 import com.sun.tools.javac.util.JCDiagnostic.Error;
  39 import com.sun.tools.javac.util.Log;
  40 import java.io.PrintWriter;
  41 import java.io.StringWriter;
  42 import java.util.ArrayDeque;
  43 import java.util.Deque;
  44 import java.util.EnumMap;
  45 import java.util.Iterator;
  46 import jdk.jshell.SourceCodeAnalysis.Completeness;
  47 import com.sun.source.tree.Tree;
  48 import static jdk.jshell.CompletenessAnalyzer.TK.*;
  49 import jdk.jshell.TaskFactory.ParseTask;
  50 import jdk.jshell.TaskFactory.Worker;
  51 import java.util.List;
  52 import java.util.function.Function;
  53 import java.util.function.Supplier;
  54 
  55 import com.sun.tools.javac.util.Names;
  56 
  57 /**
  58  * Low level scanner to determine completeness of input.
  59  * @author Robert Field
  60  */
  61 class CompletenessAnalyzer {
  62 
  63     private final ScannerFactory scannerFactory;
  64     private final JShell proc;
  65     private final Names names;
  66 
  67     private static Completeness error() {
  68         return Completeness.UNKNOWN;  // For breakpointing
  69     }
  70 
  71     static class CaInfo {
  72 
  73         CaInfo(Completeness status, int unitEndPos) {
  74             this.status = status;
  75             this.unitEndPos = unitEndPos;
  76         }
  77         final int unitEndPos;
  78         final Completeness status;
  79     }
  80 
  81     CompletenessAnalyzer(JShell proc) {
  82         this.proc = proc;
  83         Context context = new Context();
  84         Log log = CaLog.createLog(context);
  85         context.put(Log.class, log);
  86         context.put(Source.class, Source.JDK9);
  87         names = Names.instance(context);
  88         scannerFactory = ScannerFactory.instance(context);
  89     }
  90 
  91     CaInfo scan(String s) {
  92         try {
  93             Parser parser = new Parser(
  94                     () -> new Matched(scannerFactory.newScanner(s, false)),
  95                     names,
  96                     worker -> proc.taskFactory.parse(s, worker));
  97             Completeness stat = parser.parseUnit();
  98             int endPos = stat == Completeness.UNKNOWN
  99                     ? s.length()
 100                     : parser.endPos();
 101             return new CaInfo(stat, endPos);
 102         } catch (SyntaxException ex) {
 103             return new CaInfo(error(), s.length());
 104         }
 105     }
 106 
 107     @SuppressWarnings("serial")             // serialVersionUID intentionally omitted
 108     private static class SyntaxException extends RuntimeException {
 109     }
 110 
 111     private static void die() {
 112         throw new SyntaxException();
 113     }
 114 
 115     /**
 116      * Subclass of Log used by compiler API to die on error and ignore
 117      * other messages
 118      */
 119     private static class CaLog extends Log {
 120 
 121         private static CaLog createLog(Context context) {
 122             PrintWriter pw = new PrintWriter(new StringWriter());
 123             CaLog log = new CaLog(context, pw);
 124             context.put(logKey, log);
 125             return log;
 126         }
 127 
 128         private CaLog(Context context, PrintWriter pw) {
 129             super(context, pw);
 130             this.source = DiagnosticSource.NO_SOURCE;
 131         }
 132 
 133         @Override
 134         public void error(String key, Object... args) {
 135             die();
 136         }
 137 
 138         @Override
 139         public void error(int pos, Error errorKey) {
 140             die();
 141         }
 142 
 143         @Override
 144         public void error(int pos, String key, Object... args) {
 145             die();
 146         }
 147 
 148         @Override
 149         public void report(JCDiagnostic diagnostic) {
 150             // Ignore
 151         }
 152     }
 153 
 154     // Location position kinds -- a token is ...
 155     private static final int XEXPR         = 0b1;                       // OK in expression (not first)
 156     private static final int XDECL         = 0b10;                      // OK in declaration (not first)
 157     private static final int XSTMT         = 0b100;                     // OK in statement framework (not first)
 158     private static final int XEXPR1o       = 0b1000;                    // OK first in expression
 159     private static final int XDECL1o       = 0b10000;                   // OK first in declaration
 160     private static final int XSTMT1o       = 0b100000;                  // OK first or only in statement framework
 161     private static final int XEXPR1        = XEXPR1o | XEXPR;           // OK in expression (anywhere)
 162     private static final int XDECL1        = XDECL1o | XDECL;           // OK in declaration (anywhere)
 163     private static final int XSTMT1        = XSTMT1o | XSTMT;           // OK in statement framework (anywhere)
 164     private static final int XANY1         = XEXPR1o | XDECL1o | XSTMT1o;  // Mask: first in statement, declaration, or expression
 165     private static final int XTERM         = 0b100000000;               // Can terminate (last before EOF)
 166     private static final int XSTART        = 0b1000000000;              // Boundary, must be XTERM before
 167     private static final int XERRO         = 0b10000000000;             // Is an error
 168     private static final int XBRACESNEEDED = 0b100000000000;            // Expect {ANY} LBRACE
 169     private static final int XMODIFIER     = 0b1000000000000;           // Modifier
 170 
 171     /**
 172      * An extension of the compiler's TokenKind which adds our combined/processed
 173      * kinds. Also associates each TK with a union of acceptable kinds of code
 174      * position it can occupy.  For example: IDENTIFER is XEXPR1|XDECL1|XTERM,
 175      * meaning it can occur in expressions or declarations (but not in the
 176      * framework of a statement and that can be the final (terminating) token
 177      * in a snippet.
 178      * <P>
 179      * There must be a TK defined for each compiler TokenKind, an exception
 180      * will
 181      * be thrown if a TokenKind is defined and a corresponding TK is not. Add a
 182      * new TK in the appropriate category. If it is like an existing category
 183      * (e.g. a new modifier or type this may be all that is needed.  If it
 184      * is bracketing or modifies the acceptable positions of other tokens,
 185      * please closely examine the needed changes to this scanner.
 186      */
 187     static enum TK {
 188 
 189         // Special
 190         EOF(TokenKind.EOF, 0),  //
 191         ERROR(TokenKind.ERROR, XERRO),  //
 192         IDENTIFIER(TokenKind.IDENTIFIER, XEXPR1|XDECL1|XTERM),  //
 193         UNDERSCORE(TokenKind.UNDERSCORE, XERRO),  //  _
 194         CLASS(TokenKind.CLASS, XEXPR|XDECL1|XBRACESNEEDED),  //  class decl (MAPPED: DOTCLASS)
 195         RECORD(TokenKind.RECORD, XEXPR|XDECL1),  //  record decl (MAPPED: DOTCLASS)
 196         SEALED(TokenKind.SEALED, XEXPR|XDECL1),  //  sealed class decl (MAPPED: DOTCLASS)
 197         PERMITS(TokenKind.PERMITS, XEXPR|XDECL),  // permits classlist
 198         MONKEYS_AT(TokenKind.MONKEYS_AT, XEXPR|XDECL1),  //  @
 199         IMPORT(TokenKind.IMPORT, XDECL1|XSTART),  //  import -- consider declaration
 200         SEMI(TokenKind.SEMI, XSTMT1|XTERM|XSTART),  //  ;
 201 
 202         // Shouldn't see -- error
 203         PACKAGE(TokenKind.PACKAGE, XERRO),  //  package
 204         CONST(TokenKind.CONST, XERRO),  //  reserved keyword -- const
 205         GOTO(TokenKind.GOTO, XERRO),  //  reserved keyword -- goto
 206         CUSTOM(TokenKind.CUSTOM, XERRO),  // No uses
 207 
 208         // Declarations
 209         ENUM(TokenKind.ENUM, XDECL1|XBRACESNEEDED),  //  enum
 210         IMPLEMENTS(TokenKind.IMPLEMENTS, XDECL),  //  implements
 211         INTERFACE(TokenKind.INTERFACE, XDECL1|XBRACESNEEDED),  //  interface
 212         THROWS(TokenKind.THROWS, XDECL|XBRACESNEEDED),  //  throws
 213 
 214         // Primarive type names
 215         BOOLEAN(TokenKind.BOOLEAN, XEXPR1|XDECL1),  //  boolean
 216         BYTE(TokenKind.BYTE, XEXPR1|XDECL1),  //  byte
 217         CHAR(TokenKind.CHAR, XEXPR1|XDECL1),  //  char
 218         DOUBLE(TokenKind.DOUBLE, XEXPR1|XDECL1),  //  double
 219         FLOAT(TokenKind.FLOAT, XEXPR1|XDECL1),  //  float
 220         INT(TokenKind.INT, XEXPR1|XDECL1),  //  int
 221         LONG(TokenKind.LONG, XEXPR1|XDECL1),  //  long
 222         SHORT(TokenKind.SHORT, XEXPR1|XDECL1),  //  short
 223         VOID(TokenKind.VOID, XEXPR1|XDECL1),  //  void
 224         VAR(TokenKind.VAR, XEXPR1|XDECL1|XTERM),  //  var
 225 
 226         // Modifiers keywords
 227         ABSTRACT(TokenKind.ABSTRACT, XDECL1 | XMODIFIER),  //  abstract
 228         FINAL(TokenKind.FINAL, XDECL1 | XMODIFIER),  //  final
 229         NATIVE(TokenKind.NATIVE, XDECL1 | XMODIFIER),  //  native
 230         STATIC(TokenKind.STATIC, XDECL1 | XMODIFIER),  //  static
 231         STRICTFP(TokenKind.STRICTFP, XDECL1 | XMODIFIER),  //  strictfp
 232         PRIVATE(TokenKind.PRIVATE, XDECL1 | XMODIFIER),  //  private
 233         PROTECTED(TokenKind.PROTECTED, XDECL1 | XMODIFIER),  //  protected
 234         PUBLIC(TokenKind.PUBLIC, XDECL1 | XMODIFIER),  //  public
 235         TRANSIENT(TokenKind.TRANSIENT, XDECL1 | XMODIFIER),  //  transient
 236         VOLATILE(TokenKind.VOLATILE, XDECL1 | XMODIFIER),  //  volatile
 237 
 238         // Declarations and type parameters (thus expressions)
 239         EXTENDS(TokenKind.EXTENDS, XEXPR|XDECL),  //  extends
 240         COMMA(TokenKind.COMMA, XEXPR|XDECL),  //  ,
 241         AMP(TokenKind.AMP, XEXPR|XDECL, true),  //  &
 242         GT(TokenKind.GT, XEXPR|XDECL, true),  //  >
 243         LT(TokenKind.LT, XEXPR|XDECL1, true),  //  <
 244         LTLT(TokenKind.LTLT, XEXPR|XDECL1, true),  //  <<
 245         GTGT(TokenKind.GTGT, XEXPR|XDECL, true),  //  >>
 246         GTGTGT(TokenKind.GTGTGT, XEXPR|XDECL, true),  //  >>>
 247         QUES(TokenKind.QUES, XEXPR|XDECL, true),  //  ?
 248         DOT(TokenKind.DOT, XEXPR|XDECL),  //  .
 249         STAR(TokenKind.STAR, XEXPR, true),  //  * (MAPPED: DOTSTAR)
 250 
 251         // Statement keywords
 252         ASSERT(TokenKind.ASSERT, XSTMT1|XSTART),  //  assert
 253         BREAK(TokenKind.BREAK, XSTMT1|XTERM|XSTART),  //  break
 254         CATCH(TokenKind.CATCH, XSTMT1|XSTART),  //  catch
 255         CONTINUE(TokenKind.CONTINUE, XSTMT1|XTERM|XSTART),  //  continue
 256         DO(TokenKind.DO, XSTMT1|XSTART),  //  do
 257         ELSE(TokenKind.ELSE, XSTMT1|XTERM|XSTART),  //  else
 258         FINALLY(TokenKind.FINALLY, XSTMT1|XSTART),  //  finally
 259         FOR(TokenKind.FOR, XSTMT1|XSTART),  //  for
 260         IF(TokenKind.IF, XSTMT1|XSTART),  //  if
 261         RETURN(TokenKind.RETURN, XSTMT1|XTERM|XSTART),  //  return
 262         SWITCH(TokenKind.SWITCH, XSTMT1|XEXPR1),  //  switch
 263         SYNCHRONIZED(TokenKind.SYNCHRONIZED, XSTMT1|XDECL),  //  synchronized
 264         THROW(TokenKind.THROW, XSTMT1|XSTART),  //  throw
 265         TRY(TokenKind.TRY, XSTMT1|XSTART),  //  try
 266         WHILE(TokenKind.WHILE, XSTMT1|XSTART),  //  while
 267 
 268         // Statement keywords that we shouldn't see -- inside braces
 269         CASE(TokenKind.CASE, XSTMT|XSTART),  //  case
 270         DEFAULT(TokenKind.DEFAULT, XSTMT|XSTART),  //  default method, default case -- neither we should see
 271 
 272         // Expressions (can terminate)
 273         INTLITERAL(TokenKind.INTLITERAL, XEXPR1|XTERM),  //
 274         LONGLITERAL(TokenKind.LONGLITERAL, XEXPR1|XTERM),  //
 275         FLOATLITERAL(TokenKind.FLOATLITERAL, XEXPR1|XTERM),  //
 276         DOUBLELITERAL(TokenKind.DOUBLELITERAL, XEXPR1|XTERM),  //
 277         CHARLITERAL(TokenKind.CHARLITERAL, XEXPR1|XTERM),  //
 278         STRINGLITERAL(TokenKind.STRINGLITERAL, XEXPR1|XTERM),  //
 279         TRUE(TokenKind.TRUE, XEXPR1|XTERM),  //  true
 280         FALSE(TokenKind.FALSE, XEXPR1|XTERM),  //  false
 281         NULL(TokenKind.NULL, XEXPR1|XTERM),  //  null
 282         THIS(TokenKind.THIS, XEXPR1|XTERM),  //  this  -- shouldn't see
 283 
 284         // Expressions maybe terminate  //TODO handle these case separately
 285         PLUSPLUS(TokenKind.PLUSPLUS, XEXPR1|XTERM),  //  ++
 286         SUBSUB(TokenKind.SUBSUB, XEXPR1|XTERM),  //  --
 287 
 288         // Expressions cannot terminate
 289         INSTANCEOF(TokenKind.INSTANCEOF, XEXPR, true),  //  instanceof
 290         NEW(TokenKind.NEW, XEXPR1),  //  new (MAPPED: COLCOLNEW)
 291         SUPER(TokenKind.SUPER, XEXPR1|XDECL),  //  super -- shouldn't see as rec. But in type parameters
 292         ARROW(TokenKind.ARROW, XEXPR),  //  ->
 293         COLCOL(TokenKind.COLCOL, XEXPR),  //  ::
 294         LPAREN(TokenKind.LPAREN, XEXPR),  //  (
 295         RPAREN(TokenKind.RPAREN, XEXPR),  //  )
 296         LBRACE(TokenKind.LBRACE, XEXPR),  //  {
 297         RBRACE(TokenKind.RBRACE, XEXPR),  //  }
 298         LBRACKET(TokenKind.LBRACKET, XEXPR),  //  [
 299         RBRACKET(TokenKind.RBRACKET, XEXPR),  //  ]
 300         ELLIPSIS(TokenKind.ELLIPSIS, XEXPR),  //  ...
 301         EQ(TokenKind.EQ, XEXPR),  //  =
 302         BANG(TokenKind.BANG, XEXPR1),  //  !
 303         TILDE(TokenKind.TILDE, XEXPR1),  //  ~
 304         COLON(TokenKind.COLON, XEXPR|XTERM),  //  :
 305         EQEQ(TokenKind.EQEQ, XEXPR, true),  //  ==
 306         LTEQ(TokenKind.LTEQ, XEXPR, true),  //  <=
 307         GTEQ(TokenKind.GTEQ, XEXPR, true),  //  >=
 308         BANGEQ(TokenKind.BANGEQ, XEXPR, true),  //  !=
 309         AMPAMP(TokenKind.AMPAMP, XEXPR, true),  //  &&
 310         BARBAR(TokenKind.BARBAR, XEXPR, true),  //  ||
 311         PLUS(TokenKind.PLUS, XEXPR1, true),  //  +
 312         SUB(TokenKind.SUB, XEXPR1, true),  //  -
 313         SLASH(TokenKind.SLASH, XEXPR, true),  //  /
 314         BAR(TokenKind.BAR, XEXPR, true),  //  |
 315         CARET(TokenKind.CARET, XEXPR, true),  //  ^
 316         PERCENT(TokenKind.PERCENT, XEXPR, true),  //  %
 317         PLUSEQ(TokenKind.PLUSEQ, XEXPR),  //  +=
 318         SUBEQ(TokenKind.SUBEQ, XEXPR),  //  -=
 319         STAREQ(TokenKind.STAREQ, XEXPR),  //  *=
 320         SLASHEQ(TokenKind.SLASHEQ, XEXPR),  //  /=
 321         AMPEQ(TokenKind.AMPEQ, XEXPR),  //  &=
 322         BAREQ(TokenKind.BAREQ, XEXPR),  //  |=
 323         CARETEQ(TokenKind.CARETEQ, XEXPR),  //  ^=
 324         PERCENTEQ(TokenKind.PERCENTEQ, XEXPR),  //  %=
 325         LTLTEQ(TokenKind.LTLTEQ, XEXPR),  //  <<=
 326         GTGTEQ(TokenKind.GTGTEQ, XEXPR),  //  >>=
 327         GTGTGTEQ(TokenKind.GTGTGTEQ, XEXPR),  //  >>>=
 328 
 329         // combined/processed kinds
 330         UNMATCHED(XERRO),
 331         PARENS(XEXPR1|XDECL|XSTMT|XTERM),
 332         BRACKETS(XEXPR|XDECL|XTERM),
 333         BRACES(XSTMT1|XEXPR|XTERM),
 334         DOTSTAR(XDECL|XTERM),  // import foo.*
 335         COLCOLNEW(XEXPR|XTERM),  //  :: new
 336         DOTCLASS(XEXPR|XTERM),  //  class decl and .class
 337         ;
 338 
 339         static final EnumMap<TokenKind,TK> tokenKindToTKMap = new EnumMap<>(TokenKind.class);
 340 
 341         final TokenKind tokenKind;
 342         final int belongs;
 343         final boolean valueOp;
 344         Function<TK,TK> mapping;
 345 
 346         TK(int b) {
 347             this(null, b);
 348         }
 349 
 350         TK(TokenKind tokenKind, int b) {
 351             this(tokenKind, b, false);
 352         }
 353 
 354         TK(TokenKind tokenKind, int b, boolean valueOp) {
 355             this.tokenKind = tokenKind;
 356             this.belongs = b;
 357             this.valueOp = valueOp;
 358             this.mapping = null;
 359         }
 360 
 361         private static TK tokenKindToTK(TK prev, TokenKind kind) {
 362             TK tk = tokenKindToTKMap.get(kind);
 363             if (tk == null) {
 364                 System.err.printf("No corresponding %s for %s: %s\n",
 365                         TK.class.getCanonicalName(),
 366                         TokenKind.class.getCanonicalName(),
 367                         kind);
 368                 throw new InternalError("No corresponding TK for TokenKind: " + kind);
 369             }
 370             return tk.mapping != null
 371                     ? tk.mapping.apply(prev)
 372                     : tk;
 373         }
 374 
 375         boolean isOkToTerminate() {
 376             return (belongs & XTERM) != 0;
 377         }
 378 
 379         boolean isExpression() {
 380             return (belongs & XEXPR) != 0;
 381         }
 382 
 383         boolean isDeclaration() {
 384             return (belongs & XDECL) != 0;
 385         }
 386 
 387         boolean isError() {
 388             return (belongs & XERRO) != 0;
 389         }
 390 
 391         boolean isStart() {
 392             return (belongs & XSTART) != 0;
 393         }
 394 
 395         boolean isBracesNeeded() {
 396             return (belongs & XBRACESNEEDED) != 0;
 397         }
 398 
 399         boolean isModifier() {
 400             return (belongs & XMODIFIER) != 0;
 401         }
 402 
 403         /**
 404          * After construction, check that all compiler TokenKind values have
 405          * corresponding TK values.
 406          */
 407         static {
 408             for (TK tk : TK.values()) {
 409                 if (tk.tokenKind != null) {
 410                     tokenKindToTKMap.put(tk.tokenKind, tk);
 411                 }
 412             }
 413             for (TokenKind kind : TokenKind.values()) {
 414                 tokenKindToTK(null, kind); // assure they can be retrieved without error
 415             }
 416             // Mappings of disambiguated contexts
 417             STAR.mapping  = prev -> prev == DOT ? DOTSTAR : STAR;
 418             NEW.mapping   = prev -> prev == COLCOL ? COLCOLNEW : NEW;
 419             CLASS.mapping = prev -> prev == DOT ? DOTCLASS : CLASS;
 420         }
 421     }
 422 
 423     /**
 424      * A completeness scanner token.
 425      */
 426     private static class CT {
 427 
 428         /** The token kind */
 429         public final TK kind;
 430 
 431         /** The end position of this token */
 432         public final int endPos;
 433 
 434         /** The error message **/
 435         public final String message;
 436 
 437         public final Token tok;
 438 
 439         private CT(TK tk, Token tok, String msg) {
 440             this.kind = tk;
 441             this.endPos = tok.endPos;
 442             this.message = msg;
 443             this.tok = tok;
 444             //throw new InternalError(msg); /* for debugging */
 445         }
 446 
 447         private CT(TK tk, Token tok) {
 448             this.kind = tk;
 449             this.endPos = tok.endPos;
 450             this.message = null;
 451             this.tok = tok;
 452         }
 453 
 454         private CT(TK tk, int endPos) {
 455             this.kind = tk;
 456             this.endPos = endPos;
 457             this.message = null;
 458             this.tok = null;
 459         }
 460     }
 461 
 462     /**
 463      * Look for matching tokens (like parens) and other special cases, like "new"
 464      */
 465     private static class Matched implements Iterator<CT> {
 466 
 467         private final Scanner scanner;
 468         private Token current;
 469         private CT prevCT;
 470         private CT currentCT;
 471         private final Deque<Token> stack = new ArrayDeque<>();
 472 
 473         Matched(Scanner scanner) {
 474             this.scanner = scanner;
 475             advance();
 476             prevCT = currentCT = new CT(SEMI, 0); // So is valid for testing
 477         }
 478 
 479         @Override
 480         public boolean hasNext() {
 481             return currentCT.kind != EOF;
 482         }
 483 
 484         private Token advance() {
 485             Token prev = current;
 486             scanner.nextToken();
 487             current = scanner.token();
 488             return prev;
 489         }
 490 
 491         @Override
 492         public CT next() {
 493             prevCT = currentCT;
 494             currentCT = nextCT();
 495             return currentCT;
 496         }
 497 
 498         private CT match(TK tk, TokenKind open) {
 499             Token tok = advance();
 500             db("match desired-tk=%s, open=%s, seen-tok=%s", tk, open, tok.kind);
 501             if (stack.isEmpty()) {
 502                 return new CT(ERROR, tok, "Encountered '" + tok + "' with no opening '" + open + "'");
 503             }
 504             Token p = stack.pop();
 505             if (p.kind != open) {
 506                 return new CT(ERROR, tok, "No match for '" + p + "' instead encountered '" + tok + "'");
 507             }
 508             return new CT(tk, tok);
 509         }
 510 
 511         private void db(String format, Object ... args) {
 512 //            System.err.printf(format, args);
 513 //            System.err.printf(" -- stack(");
 514 //            if (stack.isEmpty()) {
 515 //
 516 //            } else {
 517 //                for (Token tok : stack) {
 518 //                    System.err.printf("%s ", tok.kind);
 519 //                }
 520 //            }
 521 //            System.err.printf(") current=%s / currentCT=%s\n", current.kind, currentCT.kind);
 522         }
 523 
 524         /**
 525          * @return the next scanner token
 526          */
 527         private CT nextCT() {
 528             // TODO Annotations?
 529             TK prevTK = currentCT.kind;
 530             while (true) {
 531                 db("nextCT");
 532                 CT ct;
 533                 switch (current.kind) {
 534                     case EOF:
 535                         db("eof");
 536                         if (stack.isEmpty()) {
 537                             ct = new CT(EOF, current);
 538                         } else {
 539                             TokenKind unmatched = stack.pop().kind;
 540                             stack.clear(); // So we will get EOF next time
 541                             ct = new CT(UNMATCHED, current, "Unmatched " + unmatched);
 542                         }
 543                         break;
 544                     case LPAREN:
 545                     case LBRACE:
 546                     case LBRACKET:
 547                         stack.push(advance());
 548                         prevTK = SEMI; // new start
 549                         continue;
 550                     case RPAREN:
 551                         ct = match(PARENS, TokenKind.LPAREN);
 552                         break;
 553                     case RBRACE:
 554                         ct = match(BRACES, TokenKind.LBRACE);
 555                         break;
 556                     case RBRACKET:
 557                         ct = match(BRACKETS, TokenKind.LBRACKET);
 558                         break;
 559                     default:
 560                         ct = new CT(TK.tokenKindToTK(prevTK, current.kind), advance());
 561                         break;
 562                 }
 563                 // Detect an error if we are at starting position and the last
 564                 // token wasn't a terminating one.  Special case: within braces,
 565                 // comma can proceed semicolon, e.g. the values list in enum
 566                 if (ct.kind.isStart() && !prevTK.isOkToTerminate() && prevTK != COMMA) {
 567                     return new CT(ERROR, current, "No '" + prevTK + "' before '" + ct.kind + "'");
 568                 }
 569                 if (stack.isEmpty() || ct.kind.isError()) {
 570                     return ct;
 571                 }
 572                 prevTK = ct.kind;
 573             }
 574         }
 575     }
 576 
 577     /**
 578      * Fuzzy parser based on token kinds
 579      */
 580     private static class Parser {
 581 
 582         private final Supplier<Matched> matchedFactory;
 583         private final Function<Worker<ParseTask, Completeness>, Completeness> parseFactory;
 584         private Matched in;
 585         private CT token;
 586         private Completeness checkResult;
 587         private final Names names;
 588 
 589         Parser(Supplier<Matched> matchedFactory,
 590                Names names,
 591                Function<Worker<ParseTask, Completeness>, Completeness> parseFactory) {
 592             this.matchedFactory = matchedFactory;
 593             this.parseFactory = parseFactory;
 594             this.names = names;
 595             resetInput();
 596         }
 597 
 598         final void resetInput() {
 599             this.in = matchedFactory.get();
 600             nextToken();
 601         }
 602 
 603         final void nextToken() {
 604             in.next();
 605             token = in.currentCT;
 606         }
 607 
 608         boolean shouldAbort(TK tk) {
 609             if (token.kind == tk) {
 610                 nextToken();
 611                 return false;
 612             }
 613             switch (token.kind) {
 614                 case EOF:
 615                     checkResult = ((tk == SEMI) && in.prevCT.kind.isOkToTerminate())
 616                             ? Completeness.COMPLETE_WITH_SEMI
 617                             : Completeness.DEFINITELY_INCOMPLETE;
 618                     return true;
 619                 case UNMATCHED:
 620                     checkResult = Completeness.DEFINITELY_INCOMPLETE;
 621                     return true;
 622                 default:
 623                     checkResult = error();
 624                     return true;
 625 
 626             }
 627         }
 628 
 629         Completeness lastly(TK tk) {
 630             if (shouldAbort(tk))  return checkResult;
 631             return Completeness.COMPLETE;
 632         }
 633 
 634         Completeness optionalFinalSemi() {
 635             if (!shouldAbort(SEMI)) return Completeness.COMPLETE;
 636             if (checkResult == Completeness.COMPLETE_WITH_SEMI) return Completeness.COMPLETE;
 637             return checkResult;
 638         }
 639 
 640         boolean shouldAbort(Completeness flags) {
 641             checkResult = flags;
 642             return flags != Completeness.COMPLETE;
 643         }
 644 
 645         public int endPos() {
 646             return in.prevCT.endPos;
 647         }
 648 
 649         public Completeness parseUnit() {
 650             //System.err.printf("%s:  belongs %o  XANY1 %o\n", token.kind, token.kind.belongs, token.kind.belongs & XANY1);
 651             switch (token.kind.belongs & XANY1) {
 652                 case XEXPR1o:
 653                     return parseExpressionOptionalSemi();
 654                 case XSTMT1o: {
 655                     Completeness stat = parseSimpleStatement();
 656                     return stat==null? error() : stat;
 657                 }
 658                 case XDECL1o:
 659                     return parseDeclaration();
 660                 case XSTMT1o | XDECL1o:
 661                 case XEXPR1o | XDECL1o:
 662                     return disambiguateDeclarationVsExpression();
 663                 case 0:
 664                     if ((token.kind.belongs & XERRO) != 0) {
 665                         return parseExpressionStatement(); // Let this gen the status
 666                     }
 667                     return error();
 668                 case XSTMT1o | XEXPR1o:
 669                     return disambiguateStatementVsExpression();
 670                 default:
 671                     throw new InternalError("Case not covered " + token.kind.belongs + " in " + token.kind);
 672             }
 673         }
 674 
 675         public Completeness parseDeclaration() {
 676             boolean isImport = token.kind == IMPORT;
 677             boolean isDatum = false;
 678             boolean afterModifiers = false;
 679             boolean isBracesNeeded = false;
 680             while (token.kind.isDeclaration()) {
 681                 isBracesNeeded |= token.kind.isBracesNeeded();
 682                 isDatum |= !afterModifiers && token.kind == TK.IDENTIFIER && token.tok.name() == names.record;
 683                 afterModifiers |= !token.kind.isModifier();
 684                 nextToken();
 685             }
 686             switch (token.kind) {
 687                 case EQ:
 688                     nextToken();
 689                     return parseExpressionStatement();
 690                 case BRACES:
 691                 case SEMI:
 692                     nextToken();
 693                     return Completeness.COMPLETE;
 694                 case UNMATCHED:
 695                     nextToken();
 696                     return Completeness.DEFINITELY_INCOMPLETE;
 697                 case EOF:
 698                     switch (in.prevCT.kind) {
 699                         case BRACES:
 700                         case SEMI:
 701                             return Completeness.COMPLETE;
 702                         case VAR:
 703                         case IDENTIFIER:
 704                             return isBracesNeeded
 705                                     ? Completeness.DEFINITELY_INCOMPLETE
 706                                     : Completeness.COMPLETE_WITH_SEMI;
 707                         case BRACKETS:
 708                             return Completeness.COMPLETE_WITH_SEMI;
 709                         case PARENS:
 710                             if (isDatum) {
 711                                 return Completeness.COMPLETE_WITH_SEMI;
 712                             } else {
 713                                 return Completeness.DEFINITELY_INCOMPLETE;
 714                             }
 715                         case DOTSTAR:
 716                             if (isImport) {
 717                                 return Completeness.COMPLETE_WITH_SEMI;
 718                             } else {
 719                                 return Completeness.UNKNOWN;
 720                             }
 721                         default:
 722                             return Completeness.DEFINITELY_INCOMPLETE;
 723                     }
 724                 default:
 725                     return error();
 726             }
 727         }
 728 
 729         public Completeness disambiguateStatementVsExpression() {
 730             if (token.kind == SWITCH) {
 731                 nextToken();
 732                 switch (token.kind) {
 733                     case PARENS:
 734                         nextToken();
 735                         break;
 736                     case UNMATCHED:
 737                         nextToken();
 738                         return Completeness.DEFINITELY_INCOMPLETE;
 739                     case EOF:
 740                         return Completeness.DEFINITELY_INCOMPLETE;
 741                     default:
 742                         return error();
 743                 }
 744                 switch (token.kind) {
 745                     case BRACES:
 746                         nextToken();
 747                         break;
 748                     case UNMATCHED:
 749                         nextToken();
 750                         return Completeness.DEFINITELY_INCOMPLETE;
 751                     case EOF:
 752                         return Completeness.DEFINITELY_INCOMPLETE;
 753                     default:
 754                         return error();
 755                 }
 756                 if (token.kind.valueOp) {
 757                     return parseExpressionOptionalSemi();
 758                 } else {
 759                     return Completeness.COMPLETE;
 760                 }
 761             } else {
 762                 throw new InternalError("Unexpected statement/expression not covered " + token.kind.belongs + " in " + token.kind);
 763             }
 764         }
 765 
 766 
 767         public Completeness disambiguateDeclarationVsExpression() {
 768             // String folding messes up position information.
 769             return parseFactory.apply(pt -> {
 770                 List<? extends Tree> units = pt.units();
 771                 if (units.isEmpty()) {
 772                     return error();
 773                 }
 774                 Tree unitTree = units.get(0);
 775                 switch (unitTree.getKind()) {
 776                     case EXPRESSION_STATEMENT:
 777                         return parseExpressionOptionalSemi();
 778                     case LABELED_STATEMENT:
 779                         if (shouldAbort(IDENTIFIER))  return checkResult;
 780                         if (shouldAbort(COLON))  return checkResult;
 781                         return parseStatement();
 782                     case VARIABLE:
 783                     case IMPORT:
 784                     case CLASS:
 785                     case ENUM:
 786                     case ANNOTATION_TYPE:
 787                     case INTERFACE:
 788                     case METHOD:
 789                         return parseDeclaration();
 790                     default:
 791                         return error();
 792                 }
 793             });
 794         }
 795 
 796         public Completeness parseExpressionStatement() {
 797             if (shouldAbort(parseExpression()))  return checkResult;
 798             return lastly(SEMI);
 799         }
 800 
 801         public Completeness parseExpressionOptionalSemi() {
 802             if (shouldAbort(parseExpression())) return checkResult;
 803             return optionalFinalSemi();
 804         }
 805 
 806         public Completeness parseExpression() {
 807             while (token.kind.isExpression())
 808                 nextToken();
 809             return Completeness.COMPLETE;
 810         }
 811 
 812         public Completeness parseStatement() {
 813             Completeness stat = parseSimpleStatement();
 814             if (stat == null) {
 815                 return parseExpressionStatement();
 816             }
 817             return stat;
 818         }
 819 
 820         /**
 821          * Statement = Block | IF ParExpression Statement [ELSE Statement] | FOR
 822          * "(" ForInitOpt ";" [Expression] ";" ForUpdateOpt ")" Statement | FOR
 823          * "(" FormalParameter : Expression ")" Statement | WHILE ParExpression
 824          * Statement | DO Statement WHILE ParExpression ";" | TRY Block (
 825          * Catches | [Catches] FinallyPart ) | TRY "(" ResourceSpecification
 826          * ";"opt ")" Block [Catches] [FinallyPart] | SWITCH ParExpression "{"
 827          * SwitchBlockStatementGroups "}" | SYNCHRONIZED ParExpression Block |
 828          * RETURN [Expression] ";" | THROW Expression ";" | BREAK [Ident] ";" |
 829          * CONTINUE [Ident] ";" | ASSERT Expression [ ":" Expression ] ";" | ";"
 830          */
 831         public Completeness parseSimpleStatement() {
 832             switch (token.kind) {
 833                 case BRACES:
 834                     return lastly(BRACES);
 835                 case IF: {
 836                     nextToken();
 837                     if (shouldAbort(PARENS))  return checkResult;
 838                     Completeness thenpart = parseStatement();
 839                     if (shouldAbort(thenpart)) return thenpart;
 840                     if (token.kind == ELSE) {
 841                         nextToken();
 842                         return parseStatement();
 843                     }
 844                     return thenpart;
 845 
 846                 }
 847                 case FOR: {
 848                     nextToken();
 849                     if (shouldAbort(PARENS))  return checkResult;
 850                     if (shouldAbort(parseStatement()))  return checkResult;
 851                     return Completeness.COMPLETE;
 852                 }
 853                 case WHILE: {
 854                     nextToken();
 855                     if (shouldAbort(PARENS))  return error();
 856                     return parseStatement();
 857                 }
 858                 case DO: {
 859                     nextToken();
 860                     switch (parseStatement()) {
 861                         case DEFINITELY_INCOMPLETE:
 862                         case CONSIDERED_INCOMPLETE:
 863                         case COMPLETE_WITH_SEMI:
 864                             return Completeness.DEFINITELY_INCOMPLETE;
 865                         case UNKNOWN:
 866                             return error();
 867                         case COMPLETE:
 868                             break;
 869                     }
 870                     if (shouldAbort(WHILE))  return checkResult;
 871                     if (shouldAbort(PARENS)) return checkResult;
 872                     return lastly(SEMI);
 873                 }
 874                 case TRY: {
 875                     boolean hasResources = false;
 876                     nextToken();
 877                     if (token.kind == PARENS) {
 878                         nextToken();
 879                         hasResources = true;
 880                     }
 881                     if (shouldAbort(BRACES))  return checkResult;
 882                     if (token.kind == CATCH || token.kind == FINALLY) {
 883                         while (token.kind == CATCH) {
 884                             if (shouldAbort(CATCH))  return checkResult;
 885                             if (shouldAbort(PARENS)) return checkResult;
 886                             if (shouldAbort(BRACES)) return checkResult;
 887                         }
 888                         if (token.kind == FINALLY) {
 889                             if (shouldAbort(FINALLY))  return checkResult;
 890                             if (shouldAbort(BRACES)) return checkResult;
 891                         }
 892                     } else if (!hasResources) {
 893                         if (token.kind == EOF) {
 894                             return Completeness.DEFINITELY_INCOMPLETE;
 895                         } else {
 896                             return error();
 897                         }
 898                     }
 899                     return Completeness.COMPLETE;
 900                 }
 901                 case SWITCH: {
 902                     nextToken();
 903                     if (shouldAbort(PARENS))  return checkResult;
 904                     return lastly(BRACES);
 905                 }
 906                 case SYNCHRONIZED: {
 907                     nextToken();
 908                     if (shouldAbort(PARENS))  return checkResult;
 909                     return lastly(BRACES);
 910                 }
 911                 case THROW: {
 912                     nextToken();
 913                     if (shouldAbort(parseExpression()))  return checkResult;
 914                     return lastly(SEMI);
 915                 }
 916                 case SEMI:
 917                     return lastly(SEMI);
 918                 case ASSERT:
 919                     nextToken();
 920                     // Crude expression parsing just happily eats the optional colon
 921                     return parseExpressionStatement();
 922                 case RETURN:
 923                 case BREAK:
 924                 case CONTINUE:
 925                     nextToken();
 926                     return parseExpressionStatement();
 927                 // What are these doing here?
 928                 case ELSE:
 929                 case FINALLY:
 930                 case CATCH:
 931                     return error();
 932                 case EOF:
 933                     return Completeness.CONSIDERED_INCOMPLETE;
 934                 default:
 935                     return null;
 936             }
 937         }
 938     }
 939 }