1 /*
  2  * Copyright (c) 2015, 2019, Oracle and/or its affiliates. All rights reserved.
  3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  4  *
  5  * This code is free software; you can redistribute it and/or modify it
  6  * under the terms of the GNU General Public License version 2 only, as
  7  * published by the Free Software Foundation.  Oracle designates this
  8  * particular file as subject to the "Classpath" exception as provided
  9  * by Oracle in the LICENSE file that accompanied this code.
 10  *
 11  * This code is distributed in the hope that it will be useful, but WITHOUT
 12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
 13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
 14  * version 2 for more details (a copy is included in the LICENSE file that
 15  * accompanied this code).
 16  *
 17  * You should have received a copy of the GNU General Public License version
 18  * 2 along with this work; if not, write to the Free Software Foundation,
 19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
 20  *
 21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
 22  * or visit www.oracle.com if you need additional information or have any
 23  * questions.
 24  */
 25 
 26 package jdk.jshell;
 27 
 28 import com.sun.tools.javac.code.Source;
 29 import com.sun.tools.javac.parser.Scanner;
 30 import com.sun.tools.javac.parser.ScannerFactory;
 31 import com.sun.tools.javac.parser.Tokens.Token;
 32 import com.sun.tools.javac.parser.Tokens.TokenKind;
 33 import com.sun.tools.javac.util.Context;
 34 import com.sun.tools.javac.util.DiagnosticSource;
 35 import com.sun.tools.javac.util.JCDiagnostic;
 36 import com.sun.tools.javac.util.JCDiagnostic.DiagnosticFlag;
 37 import com.sun.tools.javac.util.JCDiagnostic.DiagnosticPosition;
 38 import com.sun.tools.javac.util.JCDiagnostic.Error;
 39 import com.sun.tools.javac.util.Log;
 40 import java.io.PrintWriter;
 41 import java.io.StringWriter;
 42 import java.util.ArrayDeque;
 43 import java.util.Deque;
 44 import java.util.EnumMap;
 45 import java.util.Iterator;
 46 import jdk.jshell.SourceCodeAnalysis.Completeness;
 47 import com.sun.source.tree.Tree;
 48 import static jdk.jshell.CompletenessAnalyzer.TK.*;
 49 import jdk.jshell.TaskFactory.ParseTask;
 50 import jdk.jshell.TaskFactory.Worker;
 51 import java.util.List;
 52 import java.util.function.Function;
 53 import java.util.function.Supplier;
 54 
 55 import com.sun.tools.javac.util.Names;
 56 
 57 /**
 58  * Low level scanner to determine completeness of input.
 59  * @author Robert Field
 60  */
 61 class CompletenessAnalyzer {
 62 
 63     private final ScannerFactory scannerFactory;
 64     private final JShell proc;
 65     private final Names names;
 66 
 67     private static Completeness error() {
 68         return Completeness.UNKNOWN;  // For breakpointing
 69     }
 70 
 71     static class CaInfo {
 72 
 73         CaInfo(Completeness status, int unitEndPos) {
 74             this.status = status;
 75             this.unitEndPos = unitEndPos;
 76         }
 77         final int unitEndPos;
 78         final Completeness status;
 79     }
 80 
 81     CompletenessAnalyzer(JShell proc) {
 82         this.proc = proc;
 83         Context context = new Context();
 84         Log log = CaLog.createLog(context);
 85         context.put(Log.class, log);
 86         context.put(Source.class, Source.JDK9);
 87         names = Names.instance(context);
 88         scannerFactory = ScannerFactory.instance(context);
 89     }
 90 
 91     CaInfo scan(String s) {
 92         try {
 93             Parser parser = new Parser(
 94                     () -> new Matched(scannerFactory.newScanner(s, false)),
 95                     names,
 96                     worker -> proc.taskFactory.parse(s, worker));
 97             Completeness stat = parser.parseUnit();
 98             int endPos = stat == Completeness.UNKNOWN
 99                     ? s.length()
100                     : parser.endPos();
101             return new CaInfo(stat, endPos);
102         } catch (SyntaxException ex) {
103             return new CaInfo(error(), s.length());
104         }
105     }
106 
107     @SuppressWarnings("serial")             // serialVersionUID intentionally omitted
108     private static class SyntaxException extends RuntimeException {
109     }
110 
111     private static void die() {
112         throw new SyntaxException();
113     }
114 
115     /**
116      * Subclass of Log used by compiler API to die on error and ignore
117      * other messages
118      */
119     private static class CaLog extends Log {
120 
121         private static CaLog createLog(Context context) {
122             PrintWriter pw = new PrintWriter(new StringWriter());
123             CaLog log = new CaLog(context, pw);
124             context.put(logKey, log);
125             return log;
126         }
127 
128         private CaLog(Context context, PrintWriter pw) {
129             super(context, pw);
130             this.source = DiagnosticSource.NO_SOURCE;
131         }
132 
133         @Override
134         public void error(String key, Object... args) {
135             die();
136         }
137 
138         @Override
139         public void error(int pos, Error errorKey) {
140             die();
141         }
142 
143         @Override
144         public void error(int pos, String key, Object... args) {
145             die();
146         }
147 
148         @Override
149         public void report(JCDiagnostic diagnostic) {
150             // Ignore
151         }
152     }
153 
154     // Location position kinds -- a token is ...
155     private static final int XEXPR         = 0b1;                       // OK in expression (not first)
156     private static final int XDECL         = 0b10;                      // OK in declaration (not first)
157     private static final int XSTMT         = 0b100;                     // OK in statement framework (not first)
158     private static final int XEXPR1o       = 0b1000;                    // OK first in expression
159     private static final int XDECL1o       = 0b10000;                   // OK first in declaration
160     private static final int XSTMT1o       = 0b100000;                  // OK first or only in statement framework
161     private static final int XEXPR1        = XEXPR1o | XEXPR;           // OK in expression (anywhere)
162     private static final int XDECL1        = XDECL1o | XDECL;           // OK in declaration (anywhere)
163     private static final int XSTMT1        = XSTMT1o | XSTMT;           // OK in statement framework (anywhere)
164     private static final int XANY1         = XEXPR1o | XDECL1o | XSTMT1o;  // Mask: first in statement, declaration, or expression
165     private static final int XTERM         = 0b100000000;               // Can terminate (last before EOF)
166     private static final int XSTART        = 0b1000000000;              // Boundary, must be XTERM before
167     private static final int XERRO         = 0b10000000000;             // Is an error
168     private static final int XBRACESNEEDED = 0b100000000000;            // Expect {ANY} LBRACE
169     private static final int XMODIFIER     = 0b1000000000000;           // Modifier
170 
171     /**
172      * An extension of the compiler's TokenKind which adds our combined/processed
173      * kinds. Also associates each TK with a union of acceptable kinds of code
174      * position it can occupy.  For example: IDENTIFER is XEXPR1|XDECL1|XTERM,
175      * meaning it can occur in expressions or declarations (but not in the
176      * framework of a statement and that can be the final (terminating) token
177      * in a snippet.
178      * <P>
179      * There must be a TK defined for each compiler TokenKind, an exception
180      * will
181      * be thrown if a TokenKind is defined and a corresponding TK is not. Add a
182      * new TK in the appropriate category. If it is like an existing category
183      * (e.g. a new modifier or type this may be all that is needed.  If it
184      * is bracketing or modifies the acceptable positions of other tokens,
185      * please closely examine the needed changes to this scanner.
186      */
187     static enum TK {
188 
189         // Special
190         EOF(TokenKind.EOF, 0),  //
191         ERROR(TokenKind.ERROR, XERRO),  //
192         IDENTIFIER(TokenKind.IDENTIFIER, XEXPR1|XDECL1|XTERM),  //
193         UNDERSCORE(TokenKind.UNDERSCORE, XERRO),  //  _
194         CLASS(TokenKind.CLASS, XEXPR|XDECL1|XBRACESNEEDED),  //  class decl (MAPPED: DOTCLASS)
195         MONKEYS_AT(TokenKind.MONKEYS_AT, XEXPR|XDECL1),  //  @
196         IMPORT(TokenKind.IMPORT, XDECL1|XSTART),  //  import -- consider declaration
197         SEMI(TokenKind.SEMI, XSTMT1|XTERM|XSTART),  //  ;
198 
199         // Shouldn't see -- error
200         PACKAGE(TokenKind.PACKAGE, XERRO),  //  package
201         CONST(TokenKind.CONST, XERRO),  //  reserved keyword -- const
202         GOTO(TokenKind.GOTO, XERRO),  //  reserved keyword -- goto
203         CUSTOM(TokenKind.CUSTOM, XERRO),  // No uses
204 
205         // Declarations
206         ENUM(TokenKind.ENUM, XDECL1|XBRACESNEEDED),  //  enum
207         IMPLEMENTS(TokenKind.IMPLEMENTS, XDECL),  //  implements
208         INTERFACE(TokenKind.INTERFACE, XDECL1|XBRACESNEEDED),  //  interface
209         THROWS(TokenKind.THROWS, XDECL|XBRACESNEEDED),  //  throws
210 
211         // Primarive type names
212         BOOLEAN(TokenKind.BOOLEAN, XEXPR1|XDECL1),  //  boolean
213         BYTE(TokenKind.BYTE, XEXPR1|XDECL1),  //  byte
214         CHAR(TokenKind.CHAR, XEXPR1|XDECL1),  //  char
215         DOUBLE(TokenKind.DOUBLE, XEXPR1|XDECL1),  //  double
216         FLOAT(TokenKind.FLOAT, XEXPR1|XDECL1),  //  float
217         INT(TokenKind.INT, XEXPR1|XDECL1),  //  int
218         LONG(TokenKind.LONG, XEXPR1|XDECL1),  //  long
219         SHORT(TokenKind.SHORT, XEXPR1|XDECL1),  //  short
220         VOID(TokenKind.VOID, XEXPR1|XDECL1),  //  void
221 
222         // Modifiers keywords
223         ABSTRACT(TokenKind.ABSTRACT, XDECL1 | XMODIFIER),  //  abstract
224         FINAL(TokenKind.FINAL, XDECL1 | XMODIFIER),  //  final
225         NATIVE(TokenKind.NATIVE, XDECL1 | XMODIFIER),  //  native
226         STATIC(TokenKind.STATIC, XDECL1 | XMODIFIER),  //  static
227         STRICTFP(TokenKind.STRICTFP, XDECL1 | XMODIFIER),  //  strictfp
228         PRIVATE(TokenKind.PRIVATE, XDECL1 | XMODIFIER),  //  private
229         PROTECTED(TokenKind.PROTECTED, XDECL1 | XMODIFIER),  //  protected
230         PUBLIC(TokenKind.PUBLIC, XDECL1 | XMODIFIER),  //  public
231         TRANSIENT(TokenKind.TRANSIENT, XDECL1 | XMODIFIER),  //  transient
232         VOLATILE(TokenKind.VOLATILE, XDECL1 | XMODIFIER),  //  volatile
233 
234         // Declarations and type parameters (thus expressions)
235         EXTENDS(TokenKind.EXTENDS, XEXPR|XDECL),  //  extends
236         COMMA(TokenKind.COMMA, XEXPR|XDECL),  //  ,
237         AMP(TokenKind.AMP, XEXPR|XDECL, true),  //  &
238         GT(TokenKind.GT, XEXPR|XDECL, true),  //  >
239         LT(TokenKind.LT, XEXPR|XDECL1, true),  //  <
240         LTLT(TokenKind.LTLT, XEXPR|XDECL1, true),  //  <<
241         GTGT(TokenKind.GTGT, XEXPR|XDECL, true),  //  >>
242         GTGTGT(TokenKind.GTGTGT, XEXPR|XDECL, true),  //  >>>
243         QUES(TokenKind.QUES, XEXPR|XDECL, true),  //  ?
244         DOT(TokenKind.DOT, XEXPR|XDECL),  //  .
245         STAR(TokenKind.STAR, XEXPR, true),  //  * (MAPPED: DOTSTAR)
246 
247         // Statement keywords
248         ASSERT(TokenKind.ASSERT, XSTMT1|XSTART),  //  assert
249         BREAK(TokenKind.BREAK, XSTMT1|XTERM|XSTART),  //  break
250         CATCH(TokenKind.CATCH, XSTMT1|XSTART),  //  catch
251         CONTINUE(TokenKind.CONTINUE, XSTMT1|XTERM|XSTART),  //  continue
252         DO(TokenKind.DO, XSTMT1|XSTART),  //  do
253         ELSE(TokenKind.ELSE, XSTMT1|XTERM|XSTART),  //  else
254         FINALLY(TokenKind.FINALLY, XSTMT1|XSTART),  //  finally
255         FOR(TokenKind.FOR, XSTMT1|XSTART),  //  for
256         IF(TokenKind.IF, XSTMT1|XSTART),  //  if
257         RETURN(TokenKind.RETURN, XSTMT1|XTERM|XSTART),  //  return
258         SWITCH(TokenKind.SWITCH, XSTMT1|XEXPR1),  //  switch
259         SYNCHRONIZED(TokenKind.SYNCHRONIZED, XSTMT1|XDECL),  //  synchronized
260         THROW(TokenKind.THROW, XSTMT1|XSTART),  //  throw
261         TRY(TokenKind.TRY, XSTMT1|XSTART),  //  try
262         WHILE(TokenKind.WHILE, XSTMT1|XSTART),  //  while
263 
264         // Statement keywords that we shouldn't see -- inside braces
265         CASE(TokenKind.CASE, XSTMT|XSTART),  //  case
266         DEFAULT(TokenKind.DEFAULT, XSTMT|XSTART),  //  default method, default case -- neither we should see
267 
268         // Expressions (can terminate)
269         INTLITERAL(TokenKind.INTLITERAL, XEXPR1|XTERM),  //
270         LONGLITERAL(TokenKind.LONGLITERAL, XEXPR1|XTERM),  //
271         FLOATLITERAL(TokenKind.FLOATLITERAL, XEXPR1|XTERM),  //
272         DOUBLELITERAL(TokenKind.DOUBLELITERAL, XEXPR1|XTERM),  //
273         CHARLITERAL(TokenKind.CHARLITERAL, XEXPR1|XTERM),  //
274         STRINGLITERAL(TokenKind.STRINGLITERAL, XEXPR1|XTERM),  //
275         TRUE(TokenKind.TRUE, XEXPR1|XTERM),  //  true
276         FALSE(TokenKind.FALSE, XEXPR1|XTERM),  //  false
277         NULL(TokenKind.NULL, XEXPR1|XTERM),  //  null
278         THIS(TokenKind.THIS, XEXPR1|XTERM),  //  this  -- shouldn't see
279 
280         // Expressions maybe terminate  //TODO handle these case separately
281         PLUSPLUS(TokenKind.PLUSPLUS, XEXPR1|XTERM),  //  ++
282         SUBSUB(TokenKind.SUBSUB, XEXPR1|XTERM),  //  --
283 
284         // Expressions cannot terminate
285         INSTANCEOF(TokenKind.INSTANCEOF, XEXPR, true),  //  instanceof
286         NEW(TokenKind.NEW, XEXPR1),  //  new (MAPPED: COLCOLNEW)
287         SUPER(TokenKind.SUPER, XEXPR1|XDECL),  //  super -- shouldn't see as rec. But in type parameters
288         ARROW(TokenKind.ARROW, XEXPR),  //  ->
289         COLCOL(TokenKind.COLCOL, XEXPR),  //  ::
290         LPAREN(TokenKind.LPAREN, XEXPR),  //  (
291         RPAREN(TokenKind.RPAREN, XEXPR),  //  )
292         LBRACE(TokenKind.LBRACE, XEXPR),  //  {
293         RBRACE(TokenKind.RBRACE, XEXPR),  //  }
294         LBRACKET(TokenKind.LBRACKET, XEXPR),  //  [
295         RBRACKET(TokenKind.RBRACKET, XEXPR),  //  ]
296         ELLIPSIS(TokenKind.ELLIPSIS, XEXPR),  //  ...
297         EQ(TokenKind.EQ, XEXPR),  //  =
298         BANG(TokenKind.BANG, XEXPR1),  //  !
299         TILDE(TokenKind.TILDE, XEXPR1),  //  ~
300         COLON(TokenKind.COLON, XEXPR|XTERM),  //  :
301         EQEQ(TokenKind.EQEQ, XEXPR, true),  //  ==
302         LTEQ(TokenKind.LTEQ, XEXPR, true),  //  <=
303         GTEQ(TokenKind.GTEQ, XEXPR, true),  //  >=
304         BANGEQ(TokenKind.BANGEQ, XEXPR, true),  //  !=
305         AMPAMP(TokenKind.AMPAMP, XEXPR, true),  //  &&
306         BARBAR(TokenKind.BARBAR, XEXPR, true),  //  ||
307         PLUS(TokenKind.PLUS, XEXPR1, true),  //  +
308         SUB(TokenKind.SUB, XEXPR1, true),  //  -
309         SLASH(TokenKind.SLASH, XEXPR, true),  //  /
310         BAR(TokenKind.BAR, XEXPR, true),  //  |
311         CARET(TokenKind.CARET, XEXPR, true),  //  ^
312         PERCENT(TokenKind.PERCENT, XEXPR, true),  //  %
313         PLUSEQ(TokenKind.PLUSEQ, XEXPR),  //  +=
314         SUBEQ(TokenKind.SUBEQ, XEXPR),  //  -=
315         STAREQ(TokenKind.STAREQ, XEXPR),  //  *=
316         SLASHEQ(TokenKind.SLASHEQ, XEXPR),  //  /=
317         AMPEQ(TokenKind.AMPEQ, XEXPR),  //  &=
318         BAREQ(TokenKind.BAREQ, XEXPR),  //  |=
319         CARETEQ(TokenKind.CARETEQ, XEXPR),  //  ^=
320         PERCENTEQ(TokenKind.PERCENTEQ, XEXPR),  //  %=
321         LTLTEQ(TokenKind.LTLTEQ, XEXPR),  //  <<=
322         GTGTEQ(TokenKind.GTGTEQ, XEXPR),  //  >>=
323         GTGTGTEQ(TokenKind.GTGTGTEQ, XEXPR),  //  >>>=
324 
325         // combined/processed kinds
326         UNMATCHED(XERRO),
327         PARENS(XEXPR1|XDECL|XSTMT|XTERM),
328         BRACKETS(XEXPR|XDECL|XTERM),
329         BRACES(XSTMT1|XEXPR|XTERM),
330         DOTSTAR(XDECL|XTERM),  // import foo.*
331         COLCOLNEW(XEXPR|XTERM),  //  :: new
332         DOTCLASS(XEXPR|XTERM),  //  class decl and .class
333         ;
334 
335         static final EnumMap<TokenKind,TK> tokenKindToTKMap = new EnumMap<>(TokenKind.class);
336 
337         final TokenKind tokenKind;
338         final int belongs;
339         final boolean valueOp;
340         Function<TK,TK> mapping;
341 
342         TK(int b) {
343             this(null, b);
344         }
345 
346         TK(TokenKind tokenKind, int b) {
347             this(tokenKind, b, false);
348         }
349 
350         TK(TokenKind tokenKind, int b, boolean valueOp) {
351             this.tokenKind = tokenKind;
352             this.belongs = b;
353             this.valueOp = valueOp;
354             this.mapping = null;
355         }
356 
357         private static TK tokenKindToTK(TK prev, TokenKind kind) {
358             TK tk = tokenKindToTKMap.get(kind);
359             if (tk == null) {
360                 System.err.printf("No corresponding %s for %s: %s\n",
361                         TK.class.getCanonicalName(),
362                         TokenKind.class.getCanonicalName(),
363                         kind);
364                 throw new InternalError("No corresponding TK for TokenKind: " + kind);
365             }
366             return tk.mapping != null
367                     ? tk.mapping.apply(prev)
368                     : tk;
369         }
370 
371         boolean isOkToTerminate() {
372             return (belongs & XTERM) != 0;
373         }
374 
375         boolean isExpression() {
376             return (belongs & XEXPR) != 0;
377         }
378 
379         boolean isDeclaration() {
380             return (belongs & XDECL) != 0;
381         }
382 
383         boolean isError() {
384             return (belongs & XERRO) != 0;
385         }
386 
387         boolean isStart() {
388             return (belongs & XSTART) != 0;
389         }
390 
391         boolean isBracesNeeded() {
392             return (belongs & XBRACESNEEDED) != 0;
393         }
394 
395         boolean isModifier() {
396             return (belongs & XMODIFIER) != 0;
397         }
398 
399         /**
400          * After construction, check that all compiler TokenKind values have
401          * corresponding TK values.
402          */
403         static {
404             for (TK tk : TK.values()) {
405                 if (tk.tokenKind != null) {
406                     tokenKindToTKMap.put(tk.tokenKind, tk);
407                 }
408             }
409             for (TokenKind kind : TokenKind.values()) {
410                 tokenKindToTK(null, kind); // assure they can be retrieved without error
411             }
412             // Mappings of disambiguated contexts
413             STAR.mapping  = prev -> prev == DOT ? DOTSTAR : STAR;
414             NEW.mapping   = prev -> prev == COLCOL ? COLCOLNEW : NEW;
415             CLASS.mapping = prev -> prev == DOT ? DOTCLASS : CLASS;
416         }
417     }
418 
419     /**
420      * A completeness scanner token.
421      */
422     private static class CT {
423 
424         /** The token kind */
425         public final TK kind;
426 
427         /** The end position of this token */
428         public final int endPos;
429 
430         /** The error message **/
431         public final String message;
432 
433         public final Token tok;
434 
435         private CT(TK tk, Token tok, String msg) {
436             this.kind = tk;
437             this.endPos = tok.endPos;
438             this.message = msg;
439             this.tok = tok;
440             //throw new InternalError(msg); /* for debugging */
441         }
442 
443         private CT(TK tk, Token tok) {
444             this.kind = tk;
445             this.endPos = tok.endPos;
446             this.message = null;
447             this.tok = tok;
448         }
449 
450         private CT(TK tk, int endPos) {
451             this.kind = tk;
452             this.endPos = endPos;
453             this.message = null;
454             this.tok = null;
455         }
456     }
457 
458     /**
459      * Look for matching tokens (like parens) and other special cases, like "new"
460      */
461     private static class Matched implements Iterator<CT> {
462 
463         private final Scanner scanner;
464         private Token current;
465         private CT prevCT;
466         private CT currentCT;
467         private final Deque<Token> stack = new ArrayDeque<>();
468 
469         Matched(Scanner scanner) {
470             this.scanner = scanner;
471             advance();
472             prevCT = currentCT = new CT(SEMI, 0); // So is valid for testing
473         }
474 
475         @Override
476         public boolean hasNext() {
477             return currentCT.kind != EOF;
478         }
479 
480         private Token advance() {
481             Token prev = current;
482             scanner.nextToken();
483             current = scanner.token();
484             return prev;
485         }
486 
487         @Override
488         public CT next() {
489             prevCT = currentCT;
490             currentCT = nextCT();
491             return currentCT;
492         }
493 
494         private CT match(TK tk, TokenKind open) {
495             Token tok = advance();
496             db("match desired-tk=%s, open=%s, seen-tok=%s", tk, open, tok.kind);
497             if (stack.isEmpty()) {
498                 return new CT(ERROR, tok, "Encountered '" + tok + "' with no opening '" + open + "'");
499             }
500             Token p = stack.pop();
501             if (p.kind != open) {
502                 return new CT(ERROR, tok, "No match for '" + p + "' instead encountered '" + tok + "'");
503             }
504             return new CT(tk, tok);
505         }
506 
507         private void db(String format, Object ... args) {
508 //            System.err.printf(format, args);
509 //            System.err.printf(" -- stack(");
510 //            if (stack.isEmpty()) {
511 //
512 //            } else {
513 //                for (Token tok : stack) {
514 //                    System.err.printf("%s ", tok.kind);
515 //                }
516 //            }
517 //            System.err.printf(") current=%s / currentCT=%s\n", current.kind, currentCT.kind);
518         }
519 
520         /**
521          * @return the next scanner token
522          */
523         private CT nextCT() {
524             // TODO Annotations?
525             TK prevTK = currentCT.kind;
526             while (true) {
527                 db("nextCT");
528                 CT ct;
529                 switch (current.kind) {
530                     case EOF:
531                         db("eof");
532                         if (stack.isEmpty()) {
533                             ct = new CT(EOF, current);
534                         } else {
535                             TokenKind unmatched = stack.pop().kind;
536                             stack.clear(); // So we will get EOF next time
537                             ct = new CT(UNMATCHED, current, "Unmatched " + unmatched);
538                         }
539                         break;
540                     case LPAREN:
541                     case LBRACE:
542                     case LBRACKET:
543                         stack.push(advance());
544                         prevTK = SEMI; // new start
545                         continue;
546                     case RPAREN:
547                         ct = match(PARENS, TokenKind.LPAREN);
548                         break;
549                     case RBRACE:
550                         ct = match(BRACES, TokenKind.LBRACE);
551                         break;
552                     case RBRACKET:
553                         ct = match(BRACKETS, TokenKind.LBRACKET);
554                         break;
555                     default:
556                         ct = new CT(TK.tokenKindToTK(prevTK, current.kind), advance());
557                         break;
558                 }
559                 // Detect an error if we are at starting position and the last
560                 // token wasn't a terminating one.  Special case: within braces,
561                 // comma can proceed semicolon, e.g. the values list in enum
562                 if (ct.kind.isStart() && !prevTK.isOkToTerminate() && prevTK != COMMA) {
563                     return new CT(ERROR, current, "No '" + prevTK + "' before '" + ct.kind + "'");
564                 }
565                 if (stack.isEmpty() || ct.kind.isError()) {
566                     return ct;
567                 }
568                 prevTK = ct.kind;
569             }
570         }
571     }
572 
573     /**
574      * Fuzzy parser based on token kinds
575      */
576     private static class Parser {
577 
578         private final Supplier<Matched> matchedFactory;
579         private final Function<Worker<ParseTask, Completeness>, Completeness> parseFactory;
580         private Matched in;
581         private CT token;
582         private Completeness checkResult;
583         private final Names names;
584 
585         Parser(Supplier<Matched> matchedFactory,
586                Names names,
587                Function<Worker<ParseTask, Completeness>, Completeness> parseFactory) {
588             this.matchedFactory = matchedFactory;
589             this.parseFactory = parseFactory;
590             this.names = names;
591             resetInput();
592         }
593 
594         final void resetInput() {
595             this.in = matchedFactory.get();
596             nextToken();
597         }
598 
599         final void nextToken() {
600             in.next();
601             token = in.currentCT;
602         }
603 
604         boolean shouldAbort(TK tk) {
605             if (token.kind == tk) {
606                 nextToken();
607                 return false;
608             }
609             switch (token.kind) {
610                 case EOF:
611                     checkResult = ((tk == SEMI) && in.prevCT.kind.isOkToTerminate())
612                             ? Completeness.COMPLETE_WITH_SEMI
613                             : Completeness.DEFINITELY_INCOMPLETE;
614                     return true;
615                 case UNMATCHED:
616                     checkResult = Completeness.DEFINITELY_INCOMPLETE;
617                     return true;
618                 default:
619                     checkResult = error();
620                     return true;
621 
622             }
623         }
624 
625         Completeness lastly(TK tk) {
626             if (shouldAbort(tk))  return checkResult;
627             return Completeness.COMPLETE;
628         }
629 
630         Completeness optionalFinalSemi() {
631             if (!shouldAbort(SEMI)) return Completeness.COMPLETE;
632             if (checkResult == Completeness.COMPLETE_WITH_SEMI) return Completeness.COMPLETE;
633             return checkResult;
634         }
635 
636         boolean shouldAbort(Completeness flags) {
637             checkResult = flags;
638             return flags != Completeness.COMPLETE;
639         }
640 
641         public int endPos() {
642             return in.prevCT.endPos;
643         }
644 
645         public Completeness parseUnit() {
646             //System.err.printf("%s:  belongs %o  XANY1 %o\n", token.kind, token.kind.belongs, token.kind.belongs & XANY1);
647             switch (token.kind.belongs & XANY1) {
648                 case XEXPR1o:
649                     return parseExpressionOptionalSemi();
650                 case XSTMT1o: {
651                     Completeness stat = parseSimpleStatement();
652                     return stat==null? error() : stat;
653                 }
654                 case XDECL1o:
655                     return parseDeclaration();
656                 case XSTMT1o | XDECL1o:
657                 case XEXPR1o | XDECL1o:
658                     return disambiguateDeclarationVsExpression();
659                 case 0:
660                     if ((token.kind.belongs & XERRO) != 0) {
661                         return parseExpressionStatement(); // Let this gen the status
662                     }
663                     return error();
664                 case XSTMT1o | XEXPR1o:
665                     return disambiguateStatementVsExpression();
666                 default:
667                     throw new InternalError("Case not covered " + token.kind.belongs + " in " + token.kind);
668             }
669         }
670 
671         public Completeness parseDeclaration() {
672             boolean isImport = token.kind == IMPORT;
673             boolean isRecord = false;
674             boolean afterModifiers = false;
675             boolean isBracesNeeded = false;
676             while (token.kind.isDeclaration()) {
677                 isBracesNeeded |= token.kind.isBracesNeeded();
678                 isRecord |= !afterModifiers && token.kind == TK.IDENTIFIER && token.tok.name() == names.record;
679                 afterModifiers |= !token.kind.isModifier();
680                 nextToken();
681             }
682             switch (token.kind) {
683                 case EQ:
684                     nextToken();
685                     return parseExpressionStatement();
686                 case BRACES:
687                 case SEMI:
688                     nextToken();
689                     return Completeness.COMPLETE;
690                 case UNMATCHED:
691                     nextToken();
692                     return Completeness.DEFINITELY_INCOMPLETE;
693                 case EOF:
694                     switch (in.prevCT.kind) {
695                         case BRACES:
696                         case SEMI:
697                             return Completeness.COMPLETE;
698                         case IDENTIFIER:
699                             return isBracesNeeded || isRecord
700                                     ? Completeness.DEFINITELY_INCOMPLETE
701                                     : Completeness.COMPLETE_WITH_SEMI;
702                         case BRACKETS:
703                             return Completeness.COMPLETE_WITH_SEMI;
704                         case DOTSTAR:
705                             if (isImport) {
706                                 return Completeness.COMPLETE_WITH_SEMI;
707                             } else {
708                                 return Completeness.UNKNOWN;
709                             }
710                         default:
711                             return Completeness.DEFINITELY_INCOMPLETE;
712                     }
713                 default:
714                     return error();
715             }
716         }
717 
718         public Completeness disambiguateStatementVsExpression() {
719             if (token.kind == SWITCH) {
720                 nextToken();
721                 switch (token.kind) {
722                     case PARENS:
723                         nextToken();
724                         break;
725                     case UNMATCHED:
726                         nextToken();
727                         return Completeness.DEFINITELY_INCOMPLETE;
728                     case EOF:
729                         return Completeness.DEFINITELY_INCOMPLETE;
730                     default:
731                         return error();
732                 }
733                 switch (token.kind) {
734                     case BRACES:
735                         nextToken();
736                         break;
737                     case UNMATCHED:
738                         nextToken();
739                         return Completeness.DEFINITELY_INCOMPLETE;
740                     case EOF:
741                         return Completeness.DEFINITELY_INCOMPLETE;
742                     default:
743                         return error();
744                 }
745                 if (token.kind.valueOp) {
746                     return parseExpressionOptionalSemi();
747                 } else {
748                     return Completeness.COMPLETE;
749                 }
750             } else {
751                 throw new InternalError("Unexpected statement/expression not covered " + token.kind.belongs + " in " + token.kind);
752             }
753         }
754 
755 
756         public Completeness disambiguateDeclarationVsExpression() {
757             // String folding messes up position information.
758             return parseFactory.apply(pt -> {
759                 List<? extends Tree> units = pt.units();
760                 if (units.isEmpty()) {
761                     return error();
762                 }
763                 Tree unitTree = units.get(0);
764                 switch (unitTree.getKind()) {
765                     case EXPRESSION_STATEMENT:
766                         return parseExpressionOptionalSemi();
767                     case LABELED_STATEMENT:
768                         if (shouldAbort(IDENTIFIER))  return checkResult;
769                         if (shouldAbort(COLON))  return checkResult;
770                         return parseStatement();
771                     case VARIABLE:
772                     case IMPORT:
773                     case CLASS:
774                     case ENUM:
775                     case ANNOTATION_TYPE:
776                     case INTERFACE:
777                     case RECORD:
778                     case METHOD:
779                         return parseDeclaration();
780                     default:
781                         return error();
782                 }
783             });
784         }
785 
786         public Completeness parseExpressionStatement() {
787             if (shouldAbort(parseExpression()))  return checkResult;
788             return lastly(SEMI);
789         }
790 
791         public Completeness parseExpressionOptionalSemi() {
792             if (shouldAbort(parseExpression())) return checkResult;
793             return optionalFinalSemi();
794         }
795 
796         public Completeness parseExpression() {
797             while (token.kind.isExpression())
798                 nextToken();
799             return Completeness.COMPLETE;
800         }
801 
802         public Completeness parseStatement() {
803             Completeness stat = parseSimpleStatement();
804             if (stat == null) {
805                 return parseExpressionStatement();
806             }
807             return stat;
808         }
809 
810         /**
811          * Statement = Block | IF ParExpression Statement [ELSE Statement] | FOR
812          * "(" ForInitOpt ";" [Expression] ";" ForUpdateOpt ")" Statement | FOR
813          * "(" FormalParameter : Expression ")" Statement | WHILE ParExpression
814          * Statement | DO Statement WHILE ParExpression ";" | TRY Block (
815          * Catches | [Catches] FinallyPart ) | TRY "(" ResourceSpecification
816          * ";"opt ")" Block [Catches] [FinallyPart] | SWITCH ParExpression "{"
817          * SwitchBlockStatementGroups "}" | SYNCHRONIZED ParExpression Block |
818          * RETURN [Expression] ";" | THROW Expression ";" | BREAK [Ident] ";" |
819          * CONTINUE [Ident] ";" | ASSERT Expression [ ":" Expression ] ";" | ";"
820          */
821         public Completeness parseSimpleStatement() {
822             switch (token.kind) {
823                 case BRACES:
824                     return lastly(BRACES);
825                 case IF: {
826                     nextToken();
827                     if (shouldAbort(PARENS))  return checkResult;
828                     Completeness thenpart = parseStatement();
829                     if (shouldAbort(thenpart)) return thenpart;
830                     if (token.kind == ELSE) {
831                         nextToken();
832                         return parseStatement();
833                     }
834                     return thenpart;
835 
836                 }
837                 case FOR: {
838                     nextToken();
839                     if (shouldAbort(PARENS))  return checkResult;
840                     if (shouldAbort(parseStatement()))  return checkResult;
841                     return Completeness.COMPLETE;
842                 }
843                 case WHILE: {
844                     nextToken();
845                     if (shouldAbort(PARENS))  return error();
846                     return parseStatement();
847                 }
848                 case DO: {
849                     nextToken();
850                     switch (parseStatement()) {
851                         case DEFINITELY_INCOMPLETE:
852                         case CONSIDERED_INCOMPLETE:
853                         case COMPLETE_WITH_SEMI:
854                             return Completeness.DEFINITELY_INCOMPLETE;
855                         case UNKNOWN:
856                             return error();
857                         case COMPLETE:
858                             break;
859                     }
860                     if (shouldAbort(WHILE))  return checkResult;
861                     if (shouldAbort(PARENS)) return checkResult;
862                     return lastly(SEMI);
863                 }
864                 case TRY: {
865                     boolean hasResources = false;
866                     nextToken();
867                     if (token.kind == PARENS) {
868                         nextToken();
869                         hasResources = true;
870                     }
871                     if (shouldAbort(BRACES))  return checkResult;
872                     if (token.kind == CATCH || token.kind == FINALLY) {
873                         while (token.kind == CATCH) {
874                             if (shouldAbort(CATCH))  return checkResult;
875                             if (shouldAbort(PARENS)) return checkResult;
876                             if (shouldAbort(BRACES)) return checkResult;
877                         }
878                         if (token.kind == FINALLY) {
879                             if (shouldAbort(FINALLY))  return checkResult;
880                             if (shouldAbort(BRACES)) return checkResult;
881                         }
882                     } else if (!hasResources) {
883                         if (token.kind == EOF) {
884                             return Completeness.DEFINITELY_INCOMPLETE;
885                         } else {
886                             return error();
887                         }
888                     }
889                     return Completeness.COMPLETE;
890                 }
891                 case SWITCH: {
892                     nextToken();
893                     if (shouldAbort(PARENS))  return checkResult;
894                     return lastly(BRACES);
895                 }
896                 case SYNCHRONIZED: {
897                     nextToken();
898                     if (shouldAbort(PARENS))  return checkResult;
899                     return lastly(BRACES);
900                 }
901                 case THROW: {
902                     nextToken();
903                     if (shouldAbort(parseExpression()))  return checkResult;
904                     return lastly(SEMI);
905                 }
906                 case SEMI:
907                     return lastly(SEMI);
908                 case ASSERT:
909                     nextToken();
910                     // Crude expression parsing just happily eats the optional colon
911                     return parseExpressionStatement();
912                 case RETURN:
913                 case BREAK:
914                 case CONTINUE:
915                     nextToken();
916                     return parseExpressionStatement();
917                 // What are these doing here?
918                 case ELSE:
919                 case FINALLY:
920                 case CATCH:
921                     return error();
922                 case EOF:
923                     return Completeness.CONSIDERED_INCOMPLETE;
924                 default:
925                     return null;
926             }
927         }
928     }
929 }