1 /*
  2  * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.  Oracle designates this
  8  * particular file as subject to the "Classpath" exception as provided
  9  * by Oracle in the LICENSE file that accompanied this code.
 10  *
 11  * This code is distributed in the hope that it will be useful, but WITHOUT
 12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 14  * version 2 for more details (a copy is included in the LICENSE file that
 15  * accompanied this code).
 16  *
 17  * You should have received a copy of the GNU General Public License version
 18  * 2 along with this work; if not, write to the Free Software Foundation,
 19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 20  *
 21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 22  * or visit www.oracle.com if you need additional information or have any
 23  * questions.
 24  */
 25 
 26 package jdk.jshell;
 27 
 28 import com.sun.tools.javac.code.Source;
 29 import com.sun.tools.javac.parser.Scanner;
 30 import com.sun.tools.javac.parser.ScannerFactory;
 31 import com.sun.tools.javac.parser.Tokens.Token;
 32 import com.sun.tools.javac.parser.Tokens.TokenKind;
 33 import com.sun.tools.javac.util.Context;
 34 import com.sun.tools.javac.util.DiagnosticSource;
 35 import com.sun.tools.javac.util.JCDiagnostic;
 36 import com.sun.tools.javac.util.JCDiagnostic.DiagnosticFlag;
 37 import com.sun.tools.javac.util.JCDiagnostic.DiagnosticPosition;
 38 import com.sun.tools.javac.util.JCDiagnostic.Error;
 39 import com.sun.tools.javac.util.Log;
 40 import java.io.PrintWriter;
 41 import java.io.StringWriter;
 42 import java.util.ArrayDeque;
 43 import java.util.Deque;
 44 import java.util.EnumMap;
 45 import java.util.Iterator;
 46 import jdk.jshell.SourceCodeAnalysis.Completeness;
 47 import com.sun.source.tree.Tree;
 48 import static jdk.jshell.CompletenessAnalyzer.TK.*;
 49 import jdk.jshell.TaskFactory.ParseTask;
 50 import jdk.jshell.TaskFactory.Worker;
 51 import java.util.List;
 52 import java.util.function.Function;
 53 import java.util.function.Supplier;
 54 
 55 import com.sun.tools.javac.util.Names;
 56 
 57 /**
 58  * Low level scanner to determine completeness of input.
 59  * @author Robert Field
 60  */
 61 class CompletenessAnalyzer {
 62 
 63     private final ScannerFactory scannerFactory;
 64     private final JShell proc;
 65     private final Names names;
 66 
 67     private static Completeness error() {
 68         return Completeness.UNKNOWN;  // For breakpointing
 69     }
 70 
 71     static class CaInfo {
 72 
 73         CaInfo(Completeness status, int unitEndPos) {
 74             this.status = status;
 75             this.unitEndPos = unitEndPos;
 76         }
 77         final int unitEndPos;
 78         final Completeness status;
 79     }
 80 
 81     CompletenessAnalyzer(JShell proc) {
 82         this.proc = proc;
 83         Context context = new Context();
 84         Log log = CaLog.createLog(context);
 85         context.put(Log.class, log);
 86         context.put(Source.class, Source.JDK9);
 87         names = Names.instance(context);
 88         scannerFactory = ScannerFactory.instance(context);
 89     }
 90 
 91     CaInfo scan(String s) {
 92         try {
 93             Parser parser = new Parser(
 94                     () -> new Matched(scannerFactory.newScanner(s, false)),
 95                     names,
 96                     worker -> proc.taskFactory.parse(s, worker));
 97             Completeness stat = parser.parseUnit();
 98             int endPos = stat == Completeness.UNKNOWN
 99                     ? s.length()
100                     : parser.endPos();
101             return new CaInfo(stat, endPos);
102         } catch (SyntaxException ex) {
103             return new CaInfo(error(), s.length());
104         }
105     }
106 
107     @SuppressWarnings("serial")             // serialVersionUID intentionally omitted
108     private static class SyntaxException extends RuntimeException {
109     }
110 
111     private static void die() {
112         throw new SyntaxException();
113     }
114 
115     /**
116      * Subclass of Log used by compiler API to die on error and ignore
117      * other messages
118      */
119     private static class CaLog extends Log {
120 
121         private static CaLog createLog(Context context) {
122             PrintWriter pw = new PrintWriter(new StringWriter());
123             CaLog log = new CaLog(context, pw);
124             context.put(logKey, log);
125             return log;
126         }
127 
128         private CaLog(Context context, PrintWriter pw) {
129             super(context, pw);
130             this.source = DiagnosticSource.NO_SOURCE;
131         }
132 
133         @Override
134         public void error(String key, Object... args) {
135             die();
136         }
137 
138         @Override
139         public void error(int pos, Error errorKey) {
140             die();
141         }
142 
143         @Override
144         public void error(int pos, String key, Object... args) {
145             die();
146         }
147 
148         @Override
149         public void report(JCDiagnostic diagnostic) {
150             // Ignore
151         }
152     }
153 
154     // Location position kinds -- a token is ...
155     private static final int XEXPR         = 0b1;                       // OK in expression (not first)
156     private static final int XDECL         = 0b10;                      // OK in declaration (not first)
157     private static final int XSTMT         = 0b100;                     // OK in statement framework (not first)
158     private static final int XEXPR1o       = 0b1000;                    // OK first in expression
159     private static final int XDECL1o       = 0b10000;                   // OK first in declaration
160     private static final int XSTMT1o       = 0b100000;                  // OK first or only in statement framework
161     private static final int XEXPR1        = XEXPR1o | XEXPR;           // OK in expression (anywhere)
162     private static final int XDECL1        = XDECL1o | XDECL;           // OK in declaration (anywhere)
163     private static final int XSTMT1        = XSTMT1o | XSTMT;           // OK in statement framework (anywhere)
164     private static final int XANY1         = XEXPR1o | XDECL1o | XSTMT1o;  // Mask: first in statement, declaration, or expression
165     private static final int XTERM         = 0b100000000;               // Can terminate (last before EOF)
166     private static final int XSTART        = 0b1000000000;              // Boundary, must be XTERM before
167     private static final int XERRO         = 0b10000000000;             // Is an error
168     private static final int XBRACESNEEDED = 0b100000000000;            // Expect {ANY} LBRACE
169     private static final int XMODIFIER     = 0b1000000000000;           // Modifier
170 
171     /**
172      * An extension of the compiler's TokenKind which adds our combined/processed
173      * kinds. Also associates each TK with a union of acceptable kinds of code
174      * position it can occupy.  For example: IDENTIFER is XEXPR1|XDECL1|XTERM,
175      * meaning it can occur in expressions or declarations (but not in the
176      * framework of a statement and that can be the final (terminating) token
177      * in a snippet.
178      * <P>
179      * There must be a TK defined for each compiler TokenKind, an exception
180      * will
181      * be thrown if a TokenKind is defined and a corresponding TK is not. Add a
182      * new TK in the appropriate category. If it is like an existing category
183      * (e.g. a new modifier or type this may be all that is needed.  If it
184      * is bracketing or modifies the acceptable positions of other tokens,
185      * please closely examine the needed changes to this scanner.
186      */
187     static enum TK {
188 
189         // Special
190         EOF(TokenKind.EOF, 0),  //
191         ERROR(TokenKind.ERROR, XERRO),  //
192         IDENTIFIER(TokenKind.IDENTIFIER, XEXPR1|XDECL1|XTERM),  //
193         UNDERSCORE(TokenKind.UNDERSCORE, XERRO),  //  _
194         CLASS(TokenKind.CLASS, XEXPR|XDECL1|XBRACESNEEDED),  //  class decl (MAPPED: DOTCLASS)
195         MONKEYS_AT(TokenKind.MONKEYS_AT, XEXPR|XDECL1),  //  @
196         IMPORT(TokenKind.IMPORT, XDECL1|XSTART),  //  import -- consider declaration
197         SEMI(TokenKind.SEMI, XSTMT1|XTERM|XSTART),  //  ;
198 
199         // Shouldn't see -- error
200         PACKAGE(TokenKind.PACKAGE, XERRO),  //  package
201         CONST(TokenKind.CONST, XERRO),  //  reserved keyword -- const
202         GOTO(TokenKind.GOTO, XERRO),  //  reserved keyword -- goto
203         CUSTOM(TokenKind.CUSTOM, XERRO),  // No uses
204 
205         // Declarations
206         ENUM(TokenKind.ENUM, XDECL1|XBRACESNEEDED),  //  enum
207         IMPLEMENTS(TokenKind.IMPLEMENTS, XDECL),  //  implements
208         INTERFACE(TokenKind.INTERFACE, XDECL1|XBRACESNEEDED),  //  interface
209         THROWS(TokenKind.THROWS, XDECL|XBRACESNEEDED),  //  throws
210 
211         // Primarive type names
212         BOOLEAN(TokenKind.BOOLEAN, XEXPR1|XDECL1),  //  boolean
213         BYTE(TokenKind.BYTE, XEXPR1|XDECL1),  //  byte
214         CHAR(TokenKind.CHAR, XEXPR1|XDECL1),  //  char
215         DOUBLE(TokenKind.DOUBLE, XEXPR1|XDECL1),  //  double
216         FLOAT(TokenKind.FLOAT, XEXPR1|XDECL1),  //  float
217         INT(TokenKind.INT, XEXPR1|XDECL1),  //  int
218         LONG(TokenKind.LONG, XEXPR1|XDECL1),  //  long
219         SHORT(TokenKind.SHORT, XEXPR1|XDECL1),  //  short
220         VOID(TokenKind.VOID, XEXPR1|XDECL1),  //  void
221 
222         // Modifiers keywords
223         ABSTRACT(TokenKind.ABSTRACT, XDECL1 | XMODIFIER),  //  abstract
224         FINAL(TokenKind.FINAL, XDECL1 | XMODIFIER),  //  final
225         NATIVE(TokenKind.NATIVE, XDECL1 | XMODIFIER),  //  native
226         STATIC(TokenKind.STATIC, XDECL1 | XMODIFIER),  //  static
227         STRICTFP(TokenKind.STRICTFP, XDECL1 | XMODIFIER),  //  strictfp
228         PRIVATE(TokenKind.PRIVATE, XDECL1 | XMODIFIER),  //  private
229         PROTECTED(TokenKind.PROTECTED, XDECL1 | XMODIFIER),  //  protected
230         PUBLIC(TokenKind.PUBLIC, XDECL1 | XMODIFIER),  //  public
231         TRANSIENT(TokenKind.TRANSIENT, XDECL1 | XMODIFIER),  //  transient
232         VOLATILE(TokenKind.VOLATILE, XDECL1 | XMODIFIER),  //  volatile
233 
234         // Declarations and type parameters (thus expressions)
235         EXTENDS(TokenKind.EXTENDS, XEXPR|XDECL),  //  extends
236         COMMA(TokenKind.COMMA, XEXPR|XDECL),  //  ,
237         AMP(TokenKind.AMP, XEXPR|XDECL, true),  //  &
238         GT(TokenKind.GT, XEXPR|XDECL, true),  //  >
239         LT(TokenKind.LT, XEXPR|XDECL1, true),  //  <
240         LTLT(TokenKind.LTLT, XEXPR|XDECL1, true),  //  <<
241         GTGT(TokenKind.GTGT, XEXPR|XDECL, true),  //  >>
242         GTGTGT(TokenKind.GTGTGT, XEXPR|XDECL, true),  //  >>>
243         QUES(TokenKind.QUES, XEXPR|XDECL, true),  //  ?
244         DOT(TokenKind.DOT, XEXPR|XDECL),  //  .
245         STAR(TokenKind.STAR, XEXPR, true),  //  * (MAPPED: DOTSTAR)
246 
247         // Statement keywords
248         ASSERT(TokenKind.ASSERT, XSTMT1|XSTART),  //  assert
249         BREAK(TokenKind.BREAK, XSTMT1|XTERM|XSTART),  //  break
250         CATCH(TokenKind.CATCH, XSTMT1|XSTART),  //  catch
251         CONTINUE(TokenKind.CONTINUE, XSTMT1|XTERM|XSTART),  //  continue
252         DO(TokenKind.DO, XSTMT1|XSTART),  //  do
253         ELSE(TokenKind.ELSE, XSTMT1|XTERM|XSTART),  //  else
254         FINALLY(TokenKind.FINALLY, XSTMT1|XSTART),  //  finally
255         FOR(TokenKind.FOR, XSTMT1|XSTART),  //  for
256         IF(TokenKind.IF, XSTMT1|XSTART),  //  if
257         RETURN(TokenKind.RETURN, XSTMT1|XTERM|XSTART),  //  return
258         SWITCH(TokenKind.SWITCH, XSTMT1|XEXPR1),  //  switch
259         SYNCHRONIZED(TokenKind.SYNCHRONIZED, XSTMT1|XDECL),  //  synchronized
260         THROW(TokenKind.THROW, XSTMT1|XSTART),  //  throw
261         TRY(TokenKind.TRY, XSTMT1|XSTART),  //  try
262         WHILE(TokenKind.WHILE, XSTMT1|XSTART),  //  while
263         WITHFIELD(TokenKind.WITHFIELD, XEXPR),  //  withfield
264 
265         // Statement keywords that we shouldn't see -- inside braces
266         CASE(TokenKind.CASE, XSTMT|XSTART),  //  case
267         DEFAULT(TokenKind.DEFAULT, XSTMT|XSTART),  //  default method, default case -- neither we should see
268 
269         // Expressions (can terminate)
270         INTLITERAL(TokenKind.INTLITERAL, XEXPR1|XTERM),  //
271         LONGLITERAL(TokenKind.LONGLITERAL, XEXPR1|XTERM),  //
272         FLOATLITERAL(TokenKind.FLOATLITERAL, XEXPR1|XTERM),  //
273         DOUBLELITERAL(TokenKind.DOUBLELITERAL, XEXPR1|XTERM),  //
274         CHARLITERAL(TokenKind.CHARLITERAL, XEXPR1|XTERM),  //
275         STRINGLITERAL(TokenKind.STRINGLITERAL, XEXPR1|XTERM),  //
276         TRUE(TokenKind.TRUE, XEXPR1|XTERM),  //  true
277         FALSE(TokenKind.FALSE, XEXPR1|XTERM),  //  false
278         NULL(TokenKind.NULL, XEXPR1|XTERM),  //  null
279         THIS(TokenKind.THIS, XEXPR1|XTERM),  //  this  -- shouldn't see
280 
281         // Expressions maybe terminate  //TODO handle these case separately
282         PLUSPLUS(TokenKind.PLUSPLUS, XEXPR1|XTERM),  //  ++
283         SUBSUB(TokenKind.SUBSUB, XEXPR1|XTERM),  //  --
284 
285         // Expressions cannot terminate
286         INSTANCEOF(TokenKind.INSTANCEOF, XEXPR, true),  //  instanceof
287         NEW(TokenKind.NEW, XEXPR1),  //  new (MAPPED: COLCOLNEW)
288         SUPER(TokenKind.SUPER, XEXPR1|XDECL),  //  super -- shouldn't see as rec. But in type parameters
289         ARROW(TokenKind.ARROW, XEXPR),  //  ->
290         COLCOL(TokenKind.COLCOL, XEXPR),  //  ::
291         LPAREN(TokenKind.LPAREN, XEXPR),  //  (
292         RPAREN(TokenKind.RPAREN, XEXPR),  //  )
293         LBRACE(TokenKind.LBRACE, XEXPR),  //  {
294         RBRACE(TokenKind.RBRACE, XEXPR),  //  }
295         LBRACKET(TokenKind.LBRACKET, XEXPR),  //  [
296         RBRACKET(TokenKind.RBRACKET, XEXPR),  //  ]
297         ELLIPSIS(TokenKind.ELLIPSIS, XEXPR),  //  ...
298         EQ(TokenKind.EQ, XEXPR),  //  =
299         BANG(TokenKind.BANG, XEXPR1),  //  !
300         TILDE(TokenKind.TILDE, XEXPR1),  //  ~
301         COLON(TokenKind.COLON, XEXPR|XTERM),  //  :
302         EQEQ(TokenKind.EQEQ, XEXPR, true),  //  ==
303         LTEQ(TokenKind.LTEQ, XEXPR, true),  //  <=
304         GTEQ(TokenKind.GTEQ, XEXPR, true),  //  >=
305         BANGEQ(TokenKind.BANGEQ, XEXPR, true),  //  !=
306         AMPAMP(TokenKind.AMPAMP, XEXPR, true),  //  &&
307         BARBAR(TokenKind.BARBAR, XEXPR, true),  //  ||
308         PLUS(TokenKind.PLUS, XEXPR1, true),  //  +
309         SUB(TokenKind.SUB, XEXPR1, true),  //  -
310         SLASH(TokenKind.SLASH, XEXPR, true),  //  /
311         BAR(TokenKind.BAR, XEXPR, true),  //  |
312         CARET(TokenKind.CARET, XEXPR, true),  //  ^
313         PERCENT(TokenKind.PERCENT, XEXPR, true),  //  %
314         PLUSEQ(TokenKind.PLUSEQ, XEXPR),  //  +=
315         SUBEQ(TokenKind.SUBEQ, XEXPR),  //  -=
316         STAREQ(TokenKind.STAREQ, XEXPR),  //  *=
317         SLASHEQ(TokenKind.SLASHEQ, XEXPR),  //  /=
318         AMPEQ(TokenKind.AMPEQ, XEXPR),  //  &=
319         BAREQ(TokenKind.BAREQ, XEXPR),  //  |=
320         CARETEQ(TokenKind.CARETEQ, XEXPR),  //  ^=
321         PERCENTEQ(TokenKind.PERCENTEQ, XEXPR),  //  %=
322         LTLTEQ(TokenKind.LTLTEQ, XEXPR),  //  <<=
323         GTGTEQ(TokenKind.GTGTEQ, XEXPR),  //  >>=
324         GTGTGTEQ(TokenKind.GTGTGTEQ, XEXPR),  //  >>>=
325 
326         // combined/processed kinds
327         UNMATCHED(XERRO),
328         PARENS(XEXPR1|XDECL|XSTMT|XTERM),
329         BRACKETS(XEXPR|XDECL|XTERM),
330         BRACES(XSTMT1|XEXPR|XTERM),
331         DOTSTAR(XDECL|XTERM),  // import foo.*
332         COLCOLNEW(XEXPR|XTERM),  //  :: new
333         DOTCLASS(XEXPR|XTERM),  //  class decl and .class
334         ;
335 
336         static final EnumMap<TokenKind,TK> tokenKindToTKMap = new EnumMap<>(TokenKind.class);
337 
338         final TokenKind tokenKind;
339         final int belongs;
340         final boolean valueOp;
341         Function<TK,TK> mapping;
342 
343         TK(int b) {
344             this(null, b);
345         }
346 
347         TK(TokenKind tokenKind, int b) {
348             this(tokenKind, b, false);
349         }
350 
351         TK(TokenKind tokenKind, int b, boolean valueOp) {
352             this.tokenKind = tokenKind;
353             this.belongs = b;
354             this.valueOp = valueOp;
355             this.mapping = null;
356         }
357 
358         private static TK tokenKindToTK(TK prev, TokenKind kind) {
359             TK tk = tokenKindToTKMap.get(kind);
360             if (tk == null) {
361                 System.err.printf("No corresponding %s for %s: %s\n",
362                         TK.class.getCanonicalName(),
363                         TokenKind.class.getCanonicalName(),
364                         kind);
365                 throw new InternalError("No corresponding TK for TokenKind: " + kind);
366             }
367             return tk.mapping != null
368                     ? tk.mapping.apply(prev)
369                     : tk;
370         }
371 
372         boolean isOkToTerminate() {
373             return (belongs & XTERM) != 0;
374         }
375 
376         boolean isExpression() {
377             return (belongs & XEXPR) != 0;
378         }
379 
380         boolean isDeclaration() {
381             return (belongs & XDECL) != 0;
382         }
383 
384         boolean isError() {
385             return (belongs & XERRO) != 0;
386         }
387 
388         boolean isStart() {
389             return (belongs & XSTART) != 0;
390         }
391 
392         boolean isBracesNeeded() {
393             return (belongs & XBRACESNEEDED) != 0;
394         }
395 
396         boolean isModifier() {
397             return (belongs & XMODIFIER) != 0;
398         }
399 
400         /**
401          * After construction, check that all compiler TokenKind values have
402          * corresponding TK values.
403          */
404         static {
405             for (TK tk : TK.values()) {
406                 if (tk.tokenKind != null) {
407                     tokenKindToTKMap.put(tk.tokenKind, tk);
408                 }
409             }
410             for (TokenKind kind : TokenKind.values()) {
411                 tokenKindToTK(null, kind); // assure they can be retrieved without error
412             }
413             // Mappings of disambiguated contexts
414             STAR.mapping  = prev -> prev == DOT ? DOTSTAR : STAR;
415             NEW.mapping   = prev -> prev == COLCOL ? COLCOLNEW : NEW;
416             CLASS.mapping = prev -> prev == DOT ? DOTCLASS : CLASS;
417         }
418     }
419 
420     /**
421      * A completeness scanner token.
422      */
423     private static class CT {
424 
425         /** The token kind */
426         public final TK kind;
427 
428         /** The end position of this token */
429         public final int endPos;
430 
431         /** The error message **/
432         public final String message;
433 
434         public final Token tok;
435 
436         private CT(TK tk, Token tok, String msg) {
437             this.kind = tk;
438             this.endPos = tok.endPos;
439             this.message = msg;
440             this.tok = tok;
441             //throw new InternalError(msg); /* for debugging */
442         }
443 
444         private CT(TK tk, Token tok) {
445             this.kind = tk;
446             this.endPos = tok.endPos;
447             this.message = null;
448             this.tok = tok;
449         }
450 
451         private CT(TK tk, int endPos) {
452             this.kind = tk;
453             this.endPos = endPos;
454             this.message = null;
455             this.tok = null;
456         }
457     }
458 
459     /**
460      * Look for matching tokens (like parens) and other special cases, like "new"
461      */
462     private static class Matched implements Iterator<CT> {
463 
464         private final Scanner scanner;
465         private Token current;
466         private CT prevCT;
467         private CT currentCT;
468         private final Deque<Token> stack = new ArrayDeque<>();
469 
470         Matched(Scanner scanner) {
471             this.scanner = scanner;
472             advance();
473             prevCT = currentCT = new CT(SEMI, 0); // So is valid for testing
474         }
475 
476         @Override
477         public boolean hasNext() {
478             return currentCT.kind != EOF;
479         }
480 
481         private Token advance() {
482             Token prev = current;
483             scanner.nextToken();
484             current = scanner.token();
485             return prev;
486         }
487 
488         @Override
489         public CT next() {
490             prevCT = currentCT;
491             currentCT = nextCT();
492             return currentCT;
493         }
494 
495         private CT match(TK tk, TokenKind open) {
496             Token tok = advance();
497             db("match desired-tk=%s, open=%s, seen-tok=%s", tk, open, tok.kind);
498             if (stack.isEmpty()) {
499                 return new CT(ERROR, tok, "Encountered '" + tok + "' with no opening '" + open + "'");
500             }
501             Token p = stack.pop();
502             if (p.kind != open) {
503                 return new CT(ERROR, tok, "No match for '" + p + "' instead encountered '" + tok + "'");
504             }
505             return new CT(tk, tok);
506         }
507 
508         private void db(String format, Object ... args) {
509 //            System.err.printf(format, args);
510 //            System.err.printf(" -- stack(");
511 //            if (stack.isEmpty()) {
512 //
513 //            } else {
514 //                for (Token tok : stack) {
515 //                    System.err.printf("%s ", tok.kind);
516 //                }
517 //            }
518 //            System.err.printf(") current=%s / currentCT=%s\n", current.kind, currentCT.kind);
519         }
520 
521         /**
522          * @return the next scanner token
523          */
524         private CT nextCT() {
525             // TODO Annotations?
526             TK prevTK = currentCT.kind;
527             while (true) {
528                 db("nextCT");
529                 CT ct;
530                 switch (current.kind) {
531                     case EOF:
532                         db("eof");
533                         if (stack.isEmpty()) {
534                             ct = new CT(EOF, current);
535                         } else {
536                             TokenKind unmatched = stack.pop().kind;
537                             stack.clear(); // So we will get EOF next time
538                             ct = new CT(UNMATCHED, current, "Unmatched " + unmatched);
539                         }
540                         break;
541                     case LPAREN:
542                     case LBRACE:
543                     case LBRACKET:
544                         stack.push(advance());
545                         prevTK = SEMI; // new start
546                         continue;
547                     case RPAREN:
548                         ct = match(PARENS, TokenKind.LPAREN);
549                         break;
550                     case RBRACE:
551                         ct = match(BRACES, TokenKind.LBRACE);
552                         break;
553                     case RBRACKET:
554                         ct = match(BRACKETS, TokenKind.LBRACKET);
555                         break;
556                     default:
557                         ct = new CT(TK.tokenKindToTK(prevTK, current.kind), advance());
558                         break;
559                 }
560                 // Detect an error if we are at starting position and the last
561                 // token wasn't a terminating one.  Special case: within braces,
562                 // comma can proceed semicolon, e.g. the values list in enum
563                 if (ct.kind.isStart() && !prevTK.isOkToTerminate() && prevTK != COMMA) {
564                     return new CT(ERROR, current, "No '" + prevTK + "' before '" + ct.kind + "'");
565                 }
566                 if (stack.isEmpty() || ct.kind.isError()) {
567                     return ct;
568                 }
569                 prevTK = ct.kind;
570             }
571         }
572     }
573 
574     /**
575      * Fuzzy parser based on token kinds
576      */
577     private static class Parser {
578 
579         private final Supplier<Matched> matchedFactory;
580         private final Function<Worker<ParseTask, Completeness>, Completeness> parseFactory;
581         private Matched in;
582         private CT token;
583         private Completeness checkResult;
584         private final Names names;
585 
586         Parser(Supplier<Matched> matchedFactory,
587                Names names,
588                Function<Worker<ParseTask, Completeness>, Completeness> parseFactory) {
589             this.matchedFactory = matchedFactory;
590             this.parseFactory = parseFactory;
591             this.names = names;
592             resetInput();
593         }
594 
595         final void resetInput() {
596             this.in = matchedFactory.get();
597             nextToken();
598         }
599 
600         final void nextToken() {
601             in.next();
602             token = in.currentCT;
603         }
604 
605         boolean shouldAbort(TK tk) {
606             if (token.kind == tk) {
607                 nextToken();
608                 return false;
609             }
610             switch (token.kind) {
611                 case EOF:
612                     checkResult = ((tk == SEMI) && in.prevCT.kind.isOkToTerminate())
613                             ? Completeness.COMPLETE_WITH_SEMI
614                             : Completeness.DEFINITELY_INCOMPLETE;
615                     return true;
616                 case UNMATCHED:
617                     checkResult = Completeness.DEFINITELY_INCOMPLETE;
618                     return true;
619                 default:
620                     checkResult = error();
621                     return true;
622 
623             }
624         }
625 
626         Completeness lastly(TK tk) {
627             if (shouldAbort(tk))  return checkResult;
628             return Completeness.COMPLETE;
629         }
630 
631         Completeness optionalFinalSemi() {
632             if (!shouldAbort(SEMI)) return Completeness.COMPLETE;
633             if (checkResult == Completeness.COMPLETE_WITH_SEMI) return Completeness.COMPLETE;
634             return checkResult;
635         }
636 
637         boolean shouldAbort(Completeness flags) {
638             checkResult = flags;
639             return flags != Completeness.COMPLETE;
640         }
641 
642         public int endPos() {
643             return in.prevCT.endPos;
644         }
645 
646         public Completeness parseUnit() {
647             //System.err.printf("%s:  belongs %o  XANY1 %o\n", token.kind, token.kind.belongs, token.kind.belongs & XANY1);
648             switch (token.kind.belongs & XANY1) {
649                 case XEXPR1o:
650                     return parseExpressionOptionalSemi();
651                 case XSTMT1o: {
652                     Completeness stat = parseSimpleStatement();
653                     return stat==null? error() : stat;
654                 }
655                 case XDECL1o:
656                     return parseDeclaration();
657                 case XSTMT1o | XDECL1o:
658                 case XEXPR1o | XDECL1o:
659                     return disambiguateDeclarationVsExpression();
660                 case 0:
661                     if ((token.kind.belongs & XERRO) != 0) {
662                         return parseExpressionStatement(); // Let this gen the status
663                     }
664                     return error();
665                 case XSTMT1o | XEXPR1o:
666                     return disambiguateStatementVsExpression();
667                 default:
668                     throw new InternalError("Case not covered " + token.kind.belongs + " in " + token.kind);
669             }
670         }
671 
672         public Completeness parseDeclaration() {
673             boolean isImport = token.kind == IMPORT;
674             boolean isRecord = false;
675             boolean afterModifiers = false;
676             boolean isBracesNeeded = false;
677             while (token.kind.isDeclaration()) {
678                 isBracesNeeded |= token.kind.isBracesNeeded();
679                 isRecord |= !afterModifiers && token.kind == TK.IDENTIFIER && token.tok.name() == names.record;
680                 afterModifiers |= !token.kind.isModifier();
681                 nextToken();
682             }
683             switch (token.kind) {
684                 case EQ:
685                     nextToken();
686                     return parseExpressionStatement();
687                 case BRACES:
688                 case SEMI:
689                     nextToken();
690                     return Completeness.COMPLETE;
691                 case UNMATCHED:
692                     nextToken();
693                     return Completeness.DEFINITELY_INCOMPLETE;
694                 case EOF:
695                     switch (in.prevCT.kind) {
696                         case BRACES:
697                         case SEMI:
698                             return Completeness.COMPLETE;
699                         case IDENTIFIER:
700                             return isBracesNeeded || isRecord
701                                     ? Completeness.DEFINITELY_INCOMPLETE
702                                     : Completeness.COMPLETE_WITH_SEMI;
703                         case BRACKETS:
704                             return Completeness.COMPLETE_WITH_SEMI;
705                         case DOTSTAR:
706                             if (isImport) {
707                                 return Completeness.COMPLETE_WITH_SEMI;
708                             } else {
709                                 return Completeness.UNKNOWN;
710                             }
711                         default:
712                             return Completeness.DEFINITELY_INCOMPLETE;
713                     }
714                 default:
715                     return error();
716             }
717         }
718 
719         public Completeness disambiguateStatementVsExpression() {
720             if (token.kind == SWITCH) {
721                 nextToken();
722                 switch (token.kind) {
723                     case PARENS:
724                         nextToken();
725                         break;
726                     case UNMATCHED:
727                         nextToken();
728                         return Completeness.DEFINITELY_INCOMPLETE;
729                     case EOF:
730                         return Completeness.DEFINITELY_INCOMPLETE;
731                     default:
732                         return error();
733                 }
734                 switch (token.kind) {
735                     case BRACES:
736                         nextToken();
737                         break;
738                     case UNMATCHED:
739                         nextToken();
740                         return Completeness.DEFINITELY_INCOMPLETE;
741                     case EOF:
742                         return Completeness.DEFINITELY_INCOMPLETE;
743                     default:
744                         return error();
745                 }
746                 if (token.kind.valueOp) {
747                     return parseExpressionOptionalSemi();
748                 } else {
749                     return Completeness.COMPLETE;
750                 }
751             } else {
752                 throw new InternalError("Unexpected statement/expression not covered " + token.kind.belongs + " in " + token.kind);
753             }
754         }
755 
756 
757         public Completeness disambiguateDeclarationVsExpression() {
758             // String folding messes up position information.
759             return parseFactory.apply(pt -> {
760                 List<? extends Tree> units = pt.units();
761                 if (units.isEmpty()) {
762                     return error();
763                 }
764                 Tree unitTree = units.get(0);
765                 switch (unitTree.getKind()) {
766                     case EXPRESSION_STATEMENT:
767                         return parseExpressionOptionalSemi();
768                     case LABELED_STATEMENT:
769                         if (shouldAbort(IDENTIFIER))  return checkResult;
770                         if (shouldAbort(COLON))  return checkResult;
771                         return parseStatement();
772                     case VARIABLE:
773                     case IMPORT:
774                     case CLASS:
775                     case ENUM:
776                     case ANNOTATION_TYPE:
777                     case INTERFACE:
778                     case RECORD:
779                     case METHOD:
780                         return parseDeclaration();
781                     default:
782                         return error();
783                 }
784             });
785         }
786 
787         public Completeness parseExpressionStatement() {
788             if (shouldAbort(parseExpression()))  return checkResult;
789             return lastly(SEMI);
790         }
791 
792         public Completeness parseExpressionOptionalSemi() {
793             if (shouldAbort(parseExpression())) return checkResult;
794             return optionalFinalSemi();
795         }
796 
797         public Completeness parseExpression() {
798             while (token.kind.isExpression())
799                 nextToken();
800             return Completeness.COMPLETE;
801         }
802 
803         public Completeness parseStatement() {
804             Completeness stat = parseSimpleStatement();
805             if (stat == null) {
806                 return parseExpressionStatement();
807             }
808             return stat;
809         }
810 
811         /**
812          * Statement = Block | IF ParExpression Statement [ELSE Statement] | FOR
813          * "(" ForInitOpt ";" [Expression] ";" ForUpdateOpt ")" Statement | FOR
814          * "(" FormalParameter : Expression ")" Statement | WHILE ParExpression
815          * Statement | DO Statement WHILE ParExpression ";" | TRY Block (
816          * Catches | [Catches] FinallyPart ) | TRY "(" ResourceSpecification
817          * ";"opt ")" Block [Catches] [FinallyPart] | SWITCH ParExpression "{"
818          * SwitchBlockStatementGroups "}" | SYNCHRONIZED ParExpression Block |
819          * RETURN [Expression] ";" | THROW Expression ";" | BREAK [Ident] ";" |
820          * CONTINUE [Ident] ";" | ASSERT Expression [ ":" Expression ] ";" | ";"
821          */
822         public Completeness parseSimpleStatement() {
823             switch (token.kind) {
824                 case BRACES:
825                     return lastly(BRACES);
826                 case IF: {
827                     nextToken();
828                     if (shouldAbort(PARENS))  return checkResult;
829                     Completeness thenpart = parseStatement();
830                     if (shouldAbort(thenpart)) return thenpart;
831                     if (token.kind == ELSE) {
832                         nextToken();
833                         return parseStatement();
834                     }
835                     return thenpart;
836 
837                 }
838                 case FOR: {
839                     nextToken();
840                     if (shouldAbort(PARENS))  return checkResult;
841                     if (shouldAbort(parseStatement()))  return checkResult;
842                     return Completeness.COMPLETE;
843                 }
844                 case WHILE: {
845                     nextToken();
846                     if (shouldAbort(PARENS))  return error();
847                     return parseStatement();
848                 }
849                 case DO: {
850                     nextToken();
851                     switch (parseStatement()) {
852                         case DEFINITELY_INCOMPLETE:
853                         case CONSIDERED_INCOMPLETE:
854                         case COMPLETE_WITH_SEMI:
855                             return Completeness.DEFINITELY_INCOMPLETE;
856                         case UNKNOWN:
857                             return error();
858                         case COMPLETE:
859                             break;
860                     }
861                     if (shouldAbort(WHILE))  return checkResult;
862                     if (shouldAbort(PARENS)) return checkResult;
863                     return lastly(SEMI);
864                 }
865                 case TRY: {
866                     boolean hasResources = false;
867                     nextToken();
868                     if (token.kind == PARENS) {
869                         nextToken();
870                         hasResources = true;
871                     }
872                     if (shouldAbort(BRACES))  return checkResult;
873                     if (token.kind == CATCH || token.kind == FINALLY) {
874                         while (token.kind == CATCH) {
875                             if (shouldAbort(CATCH))  return checkResult;
876                             if (shouldAbort(PARENS)) return checkResult;
877                             if (shouldAbort(BRACES)) return checkResult;
878                         }
879                         if (token.kind == FINALLY) {
880                             if (shouldAbort(FINALLY))  return checkResult;
881                             if (shouldAbort(BRACES)) return checkResult;
882                         }
883                     } else if (!hasResources) {
884                         if (token.kind == EOF) {
885                             return Completeness.DEFINITELY_INCOMPLETE;
886                         } else {
887                             return error();
888                         }
889                     }
890                     return Completeness.COMPLETE;
891                 }
892                 case SWITCH: {
893                     nextToken();
894                     if (shouldAbort(PARENS))  return checkResult;
895                     return lastly(BRACES);
896                 }
897                 case SYNCHRONIZED: {
898                     nextToken();
899                     if (shouldAbort(PARENS))  return checkResult;
900                     return lastly(BRACES);
901                 }
902                 case THROW: {
903                     nextToken();
904                     if (shouldAbort(parseExpression()))  return checkResult;
905                     return lastly(SEMI);
906                 }
907                 case SEMI:
908                     return lastly(SEMI);
909                 case ASSERT:
910                     nextToken();
911                     // Crude expression parsing just happily eats the optional colon
912                     return parseExpressionStatement();
913                 case RETURN:
914                 case BREAK:
915                 case CONTINUE:
916                     nextToken();
917                     return parseExpressionStatement();
918                 // What are these doing here?
919                 case ELSE:
920                 case FINALLY:
921                 case CATCH:
922                     return error();
923                 case EOF:
924                     return Completeness.CONSIDERED_INCOMPLETE;
925                 default:
926                     return null;
927             }
928         }
929     }
930 }