1 /*
   2  * Copyright (c) 1999, 2018, Oracle and/or its affiliates. All rights reserved.
   3  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   4  *
   5  * This code is free software; you can redistribute it and/or modify it
   6  * under the terms of the GNU General Public License version 2 only, as
   7  * published by the Free Software Foundation.  Oracle designates this
   8  * particular file as subject to the "Classpath" exception as provided
   9  * by Oracle in the LICENSE file that accompanied this code.
  10  *
  11  * This code is distributed in the hope that it will be useful, but WITHOUT
  12  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  13  * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  14  * version 2 for more details (a copy is included in the LICENSE file that
  15  * accompanied this code).
  16  *
  17  * You should have received a copy of the GNU General Public License version
  18  * 2 along with this work; if not, write to the Free Software Foundation,
  19  * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
  20  *
  21  * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
  22  * or visit www.oracle.com if you need additional information or have any
  23  * questions.
  24  */
  25 
  26 package com.sun.tools.javac.parser;
  27 
  28 import com.sun.tools.javac.code.Preview;
  29 import com.sun.tools.javac.code.Source;
  30 import com.sun.tools.javac.code.Source.Feature;
  31 import com.sun.tools.javac.parser.Tokens.Comment.CommentStyle;
  32 import com.sun.tools.javac.resources.CompilerProperties.Errors;
  33 import com.sun.tools.javac.util.*;
  34 import com.sun.tools.javac.util.JCDiagnostic.DiagnosticFlag;
  35 
  36 import java.nio.CharBuffer;
  37 
  38 import static com.sun.tools.javac.parser.Tokens.*;
  39 import static com.sun.tools.javac.util.LayoutCharacters.*;
  40 
  41 /** The lexical analyzer maps an input stream consisting of
  42  *  ASCII characters and Unicode escapes into a token sequence.
  43  *
  44  *  <p><b>This is NOT part of any supported API.
  45  *  If you write code that depends on this, you do so at your own risk.
  46  *  This code and its internal interfaces are subject to change or
  47  *  deletion without notice.</b>
  48  */
  49 public class JavaTokenizer {
  50 
  51     private static final boolean scannerDebug = false;
  52 
  53     /** The source language setting.
  54      */
  55     private Source source;
  56 
  57     /** The preview language setting. */
  58     private Preview preview;
  59 
  60     /** The log to be used for error reporting.
  61      */
  62     private final Log log;
  63 
  64     /** The token factory. */
  65     private final Tokens tokens;
  66 
  67     /** The token kind, set by nextToken().
  68      */
  69     protected TokenKind tk;
  70 
  71     /** The token's radix, set by nextToken().
  72      */
  73     protected int radix;
  74 
  75     /** The token's name, set by nextToken().
  76      */
  77     protected Name name;
  78 
  79     /** The position where a lexical error occurred;
  80      */
  81     protected int errPos = Position.NOPOS;
  82 
  83     /** The Unicode reader (low-level stream reader).
  84      */
  85     protected UnicodeReader reader;
  86 
  87     protected ScannerFactory fac;
  88 
  89     private static final boolean hexFloatsWork = hexFloatsWork();
  90     private static boolean hexFloatsWork() {
  91         try {
  92             Float.valueOf("0x1.0p1");
  93             return true;
  94         } catch (NumberFormatException ex) {
  95             return false;
  96         }
  97     }
  98 
  99     /**
 100      * Create a scanner from the input array.  This method might
 101      * modify the array.  To avoid copying the input array, ensure
 102      * that {@code inputLength < input.length} or
 103      * {@code input[input.length -1]} is a white space character.
 104      *
 105      * @param fac the factory which created this Scanner
 106      * @param buf the input, might be modified
 107      * Must be positive and less than or equal to input.length.
 108      */
 109     protected JavaTokenizer(ScannerFactory fac, CharBuffer buf) {
 110         this(fac, new UnicodeReader(fac, buf));
 111     }
 112 
 113     protected JavaTokenizer(ScannerFactory fac, char[] buf, int inputLength) {
 114         this(fac, new UnicodeReader(fac, buf, inputLength));
 115     }
 116 
 117     protected JavaTokenizer(ScannerFactory fac, UnicodeReader reader) {
 118         this.fac = fac;
 119         this.log = fac.log;
 120         this.tokens = fac.tokens;
 121         this.source = fac.source;
 122         this.preview = fac.preview;
 123         this.reader = reader;
 124     }
 125 
 126     protected void checkSourceLevel(int pos, Feature feature) {
 127         if (preview.isPreview(feature) && !preview.isEnabled()) {
 128             //preview feature without --preview flag, error
 129             lexError(DiagnosticFlag.SOURCE_LEVEL, pos, preview.disabledError(feature));
 130         } else if (!feature.allowedInSource(source)) {
 131             //incompatible source level, error
 132             lexError(DiagnosticFlag.SOURCE_LEVEL, pos, feature.error(source.name));
 133         } else if (preview.isPreview(feature)) {
 134             //use of preview feature, warn
 135             preview.warnPreview(pos, feature);
 136         }
 137     }
 138 
 139     /** Report an error at the given position using the provided arguments.
 140      */
 141     protected void lexError(int pos, JCDiagnostic.Error key) {
 142         log.error(pos, key);
 143         tk = TokenKind.ERROR;
 144         errPos = pos;
 145     }
 146 
 147     protected void lexError(DiagnosticFlag flags, int pos, JCDiagnostic.Error key) {
 148         log.error(flags, pos, key);
 149         tk = TokenKind.ERROR;
 150         errPos = pos;
 151     }
 152 
 153     /** Read next character in character or string literal and copy into sbuf.
 154      */
 155     private void scanLitChar(int pos) {
 156         if (reader.ch == '\\') {
 157             if (reader.peekChar() == '\\' && !reader.isUnicode()) {
 158                 reader.skipChar();
 159                 reader.putChar('\\', true);
 160             } else {
 161                 reader.scanChar();
 162                 switch (reader.ch) {
 163                 case '0': case '1': case '2': case '3':
 164                 case '4': case '5': case '6': case '7':
 165                     char leadch = reader.ch;
 166                     int oct = reader.digit(pos, 8);
 167                     reader.scanChar();
 168                     if ('0' <= reader.ch && reader.ch <= '7') {
 169                         oct = oct * 8 + reader.digit(pos, 8);
 170                         reader.scanChar();
 171                         if (leadch <= '3' && '0' <= reader.ch && reader.ch <= '7') {
 172                             oct = oct * 8 + reader.digit(pos, 8);
 173                             reader.scanChar();
 174                         }
 175                     }
 176                     reader.putChar((char)oct);
 177                     break;
 178                 case 'b':
 179                     reader.putChar('\b', true); break;
 180                 case 't':
 181                     reader.putChar('\t', true); break;
 182                 case 'n':
 183                     reader.putChar('\n', true); break;
 184                 case 'f':
 185                     reader.putChar('\f', true); break;
 186                 case 'r':
 187                     reader.putChar('\r', true); break;
 188                 case '\'':
 189                     reader.putChar('\'', true); break;
 190                 case '\"':
 191                     reader.putChar('\"', true); break;
 192                 case '\\':
 193                     reader.putChar('\\', true); break;
 194                 default:
 195                     lexError(reader.bp, Errors.IllegalEscChar);
 196                 }
 197             }
 198         } else if (reader.bp != reader.buflen) {
 199             reader.putChar(true);
 200         }
 201     }
 202 
 203     private void scanDigits(int pos, int digitRadix) {
 204         char saveCh;
 205         int savePos;
 206         do {
 207             if (reader.ch != '_') {
 208                 reader.putChar(false);
 209             }
 210             saveCh = reader.ch;
 211             savePos = reader.bp;
 212             reader.scanChar();
 213         } while (reader.digit(pos, digitRadix) >= 0 || reader.ch == '_');
 214         if (saveCh == '_')
 215             lexError(savePos, Errors.IllegalUnderscore);
 216     }
 217 
 218     /** Read fractional part of hexadecimal floating point number.
 219      */
 220     private void scanHexExponentAndSuffix(int pos) {
 221         if (reader.ch == 'p' || reader.ch == 'P') {
 222             reader.putChar(true);
 223             skipIllegalUnderscores();
 224             if (reader.ch == '+' || reader.ch == '-') {
 225                 reader.putChar(true);
 226             }
 227             skipIllegalUnderscores();
 228             if (reader.digit(pos, 10) >= 0) {
 229                 scanDigits(pos, 10);
 230                 if (!hexFloatsWork)
 231                     lexError(pos, Errors.UnsupportedCrossFpLit);
 232             } else
 233                 lexError(pos, Errors.MalformedFpLit);
 234         } else {
 235             lexError(pos, Errors.MalformedFpLit);
 236         }
 237         if (reader.ch == 'f' || reader.ch == 'F') {
 238             reader.putChar(true);
 239             tk = TokenKind.FLOATLITERAL;
 240             radix = 16;
 241         } else {
 242             if (reader.ch == 'd' || reader.ch == 'D') {
 243                 reader.putChar(true);
 244             }
 245             tk = TokenKind.DOUBLELITERAL;
 246             radix = 16;
 247         }
 248     }
 249 
 250     /** Read fractional part of floating point number.
 251      */
 252     private void scanFraction(int pos) {
 253         skipIllegalUnderscores();
 254         if (reader.digit(pos, 10) >= 0) {
 255             scanDigits(pos, 10);
 256         }
 257         int sp1 = reader.sp;
 258         if (reader.ch == 'e' || reader.ch == 'E') {
 259             reader.putChar(true);
 260             skipIllegalUnderscores();
 261             if (reader.ch == '+' || reader.ch == '-') {
 262                 reader.putChar(true);
 263             }
 264             skipIllegalUnderscores();
 265             if (reader.digit(pos, 10) >= 0) {
 266                 scanDigits(pos, 10);
 267                 return;
 268             }
 269             lexError(pos, Errors.MalformedFpLit);
 270             reader.sp = sp1;
 271         }
 272     }
 273 
 274     /** Read fractional part and 'd' or 'f' suffix of floating point number.
 275      */
 276     private void scanFractionAndSuffix(int pos) {
 277         radix = 10;
 278         scanFraction(pos);
 279         if (reader.ch == 'f' || reader.ch == 'F') {
 280             reader.putChar(true);
 281             tk = TokenKind.FLOATLITERAL;
 282         } else {
 283             if (reader.ch == 'd' || reader.ch == 'D') {
 284                 reader.putChar(true);
 285             }
 286             tk = TokenKind.DOUBLELITERAL;
 287         }
 288     }
 289 
 290     /** Read fractional part and 'd' or 'f' suffix of floating point number.
 291      */
 292     private void scanHexFractionAndSuffix(int pos, boolean seendigit) {
 293         radix = 16;
 294         Assert.check(reader.ch == '.');
 295         reader.putChar(true);
 296         skipIllegalUnderscores();
 297         if (reader.digit(pos, 16) >= 0) {
 298             seendigit = true;
 299             scanDigits(pos, 16);
 300         }
 301         if (!seendigit)
 302             lexError(pos, Errors.InvalidHexNumber);
 303         else
 304             scanHexExponentAndSuffix(pos);
 305     }
 306 
 307     private void skipIllegalUnderscores() {
 308         if (reader.ch == '_') {
 309             lexError(reader.bp, Errors.IllegalUnderscore);
 310             while (reader.ch == '_')
 311                 reader.scanChar();
 312         }
 313     }
 314 
 315     /** Read a number.
 316      *  @param radix  The radix of the number; one of 2, 8, 10, 16.
 317      */
 318     private void scanNumber(int pos, int radix) {
 319         // for octal, allow base-10 digit in case it's a float literal
 320         this.radix = radix;
 321         int digitRadix = (radix == 8 ? 10 : radix);
 322         int firstDigit = reader.digit(pos, Math.max(10, digitRadix));
 323         boolean seendigit = firstDigit >= 0;
 324         boolean seenValidDigit = firstDigit >= 0 && firstDigit < digitRadix;
 325         if (seendigit) {
 326             scanDigits(pos, digitRadix);
 327         }
 328         if (radix == 16 && reader.ch == '.') {
 329             scanHexFractionAndSuffix(pos, seendigit);
 330         } else if (seendigit && radix == 16 && (reader.ch == 'p' || reader.ch == 'P')) {
 331             scanHexExponentAndSuffix(pos);
 332         } else if (digitRadix == 10 && reader.ch == '.') {
 333             reader.putChar(true);
 334             scanFractionAndSuffix(pos);
 335         } else if (digitRadix == 10 &&
 336                    (reader.ch == 'e' || reader.ch == 'E' ||
 337                     reader.ch == 'f' || reader.ch == 'F' ||
 338                     reader.ch == 'd' || reader.ch == 'D')) {
 339             scanFractionAndSuffix(pos);
 340         } else {
 341             if (!seenValidDigit) {
 342                 switch (radix) {
 343                 case 2:
 344                     lexError(pos, Errors.InvalidBinaryNumber);
 345                     break;
 346                 case 16:
 347                     lexError(pos, Errors.InvalidHexNumber);
 348                     break;
 349                 }
 350             }
 351             if (reader.ch == 'l' || reader.ch == 'L') {
 352                 reader.scanChar();
 353                 tk = TokenKind.LONGLITERAL;
 354             } else {
 355                 tk = TokenKind.INTLITERAL;
 356             }
 357         }
 358     }
 359 
 360     /** Read an identifier.
 361      */
 362     private void scanIdent() {
 363         boolean isJavaIdentifierPart;
 364         char high;
 365         reader.putChar(true);
 366         do {
 367             switch (reader.ch) {
 368             case 'A': case 'B': case 'C': case 'D': case 'E':
 369             case 'F': case 'G': case 'H': case 'I': case 'J':
 370             case 'K': case 'L': case 'M': case 'N': case 'O':
 371             case 'P': case 'Q': case 'R': case 'S': case 'T':
 372             case 'U': case 'V': case 'W': case 'X': case 'Y':
 373             case 'Z':
 374             case 'a': case 'b': case 'c': case 'd': case 'e':
 375             case 'f': case 'g': case 'h': case 'i': case 'j':
 376             case 'k': case 'l': case 'm': case 'n': case 'o':
 377             case 'p': case 'q': case 'r': case 's': case 't':
 378             case 'u': case 'v': case 'w': case 'x': case 'y':
 379             case 'z':
 380             case '$': case '_':
 381             case '0': case '1': case '2': case '3': case '4':
 382             case '5': case '6': case '7': case '8': case '9':
 383                 break;
 384             case '\u0000': case '\u0001': case '\u0002': case '\u0003':
 385             case '\u0004': case '\u0005': case '\u0006': case '\u0007':
 386             case '\u0008': case '\u000E': case '\u000F': case '\u0010':
 387             case '\u0011': case '\u0012': case '\u0013': case '\u0014':
 388             case '\u0015': case '\u0016': case '\u0017':
 389             case '\u0018': case '\u0019': case '\u001B':
 390             case '\u007F':
 391                 reader.scanChar();
 392                 continue;
 393             case '\u001A': // EOI is also a legal identifier part
 394                 if (reader.bp >= reader.buflen) {
 395                     name = reader.name();
 396                     tk = tokens.lookupKind(name);
 397                     return;
 398                 }
 399                 reader.scanChar();
 400                 continue;
 401             default:
 402                 if (reader.ch < '\u0080') {
 403                     // all ASCII range chars already handled, above
 404                     isJavaIdentifierPart = false;
 405                 } else {
 406                     if (Character.isIdentifierIgnorable(reader.ch)) {
 407                         reader.scanChar();
 408                         continue;
 409                     } else {
 410                         int codePoint = reader.peekSurrogates();
 411                         if (codePoint >= 0) {
 412                             if (isJavaIdentifierPart = Character.isJavaIdentifierPart(codePoint)) {
 413                                 reader.putChar(true);
 414                             }
 415                         } else {
 416                             isJavaIdentifierPart = Character.isJavaIdentifierPart(reader.ch);
 417                         }
 418                     }
 419                 }
 420                 if (!isJavaIdentifierPart) {
 421                     name = reader.name();
 422                     tk = tokens.lookupKind(name);
 423                     return;
 424                 }
 425             }
 426             reader.putChar(true);
 427         } while (true);
 428     }
 429 
 430     /** Return true if reader.ch can be part of an operator.
 431      */
 432     private boolean isSpecial(char ch) {
 433         switch (ch) {
 434         case '!': case '%': case '&': case '*': case '?':
 435         case '+': case '-': case ':': case '<': case '=':
 436         case '>': case '^': case '|': case '~':
 437         case '@':
 438             return true;
 439         default:
 440             return false;
 441         }
 442     }
 443 
 444     /** Read longest possible sequence of special characters and convert
 445      *  to token.
 446      */
 447     private void scanOperator() {
 448         while (true) {
 449             reader.putChar(false);
 450             Name newname = reader.name();
 451             TokenKind tk1 = tokens.lookupKind(newname);
 452             if (tk1 == TokenKind.IDENTIFIER) {
 453                 reader.sp--;
 454                 break;
 455             }
 456             tk = tk1;
 457             reader.scanChar();
 458             if (!isSpecial(reader.ch)) break;
 459         }
 460     }
 461 
 462     /** Read token.
 463      */
 464     public Token readToken() {
 465 
 466         reader.sp = 0;
 467         name = null;
 468         radix = 0;
 469 
 470         int pos = 0;
 471         int endPos = 0;
 472         List<Comment> comments = null;
 473 
 474         try {
 475             loop: while (true) {
 476                 pos = reader.bp;
 477                 switch (reader.ch) {
 478                 case ' ': // (Spec 3.6)
 479                 case '\t': // (Spec 3.6)
 480                 case FF: // (Spec 3.6)
 481                     do {
 482                         reader.scanChar();
 483                     } while (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF);
 484                     processWhiteSpace(pos, reader.bp);
 485                     break;
 486                 case LF: // (Spec 3.4)
 487                     reader.scanChar();
 488                     processLineTerminator(pos, reader.bp);
 489                     break;
 490                 case CR: // (Spec 3.4)
 491                     reader.scanChar();
 492                     if (reader.ch == LF) {
 493                         reader.scanChar();
 494                     }
 495                     processLineTerminator(pos, reader.bp);
 496                     break;
 497                 case 'A': case 'B': case 'C': case 'D': case 'E':
 498                 case 'F': case 'G': case 'H': case 'I': case 'J':
 499                 case 'K': case 'L': case 'M': case 'N': case 'O':
 500                 case 'P': case 'Q': case 'R': case 'S': case 'T':
 501                 case 'U': case 'V': case 'W': case 'X': case 'Y':
 502                 case 'Z':
 503                 case 'a': case 'b': case 'c': case 'd': case 'e':
 504                 case 'f': case 'g': case 'h': case 'i': case 'j':
 505                 case 'k': case 'l': case 'm': case 'n': case 'o':
 506                 case 'p': case 'q': case 'r': case 's': case 't':
 507                 case 'u': case 'v': case 'w': case 'x': case 'y':
 508                 case 'z':
 509                 case '$': case '_':
 510                     scanIdent();
 511                     break loop;
 512                 case '0':
 513                     reader.scanChar();
 514                     if (reader.ch == 'x' || reader.ch == 'X') {
 515                         reader.scanChar();
 516                         skipIllegalUnderscores();
 517                         scanNumber(pos, 16);
 518                     } else if (reader.ch == 'b' || reader.ch == 'B') {
 519                         reader.scanChar();
 520                         skipIllegalUnderscores();
 521                         scanNumber(pos, 2);
 522                     } else {
 523                         reader.putChar('0');
 524                         if (reader.ch == '_') {
 525                             int savePos = reader.bp;
 526                             do {
 527                                 reader.scanChar();
 528                             } while (reader.ch == '_');
 529                             if (reader.digit(pos, 10) < 0) {
 530                                 lexError(savePos, Errors.IllegalUnderscore);
 531                             }
 532                         }
 533                         scanNumber(pos, 8);
 534                     }
 535                     break loop;
 536                 case '1': case '2': case '3': case '4':
 537                 case '5': case '6': case '7': case '8': case '9':
 538                     scanNumber(pos, 10);
 539                     break loop;
 540                 case '.':
 541                     reader.scanChar();
 542                     if (reader.digit(pos, 10) >= 0) {
 543                         reader.putChar('.');
 544                         scanFractionAndSuffix(pos);
 545                     } else if (reader.ch == '.') {
 546                         int savePos = reader.bp;
 547                         reader.putChar('.'); reader.putChar('.', true);
 548                         if (reader.ch == '.') {
 549                             reader.scanChar();
 550                             reader.putChar('.');
 551                             tk = TokenKind.ELLIPSIS;
 552                         } else {
 553                             lexError(savePos, Errors.IllegalDot);
 554                         }
 555                     } else {
 556                         tk = TokenKind.DOT;
 557                     }
 558                     break loop;
 559                 case ',':
 560                     reader.scanChar(); tk = TokenKind.COMMA; break loop;
 561                 case ';':
 562                     reader.scanChar(); tk = TokenKind.SEMI; break loop;
 563                 case '(':
 564                     reader.scanChar(); tk = TokenKind.LPAREN; break loop;
 565                 case ')':
 566                     reader.scanChar(); tk = TokenKind.RPAREN; break loop;
 567                 case '[':
 568                     reader.scanChar(); tk = TokenKind.LBRACKET; break loop;
 569                 case ']':
 570                     reader.scanChar(); tk = TokenKind.RBRACKET; break loop;
 571                 case '{':
 572                     reader.scanChar(); tk = TokenKind.LBRACE; break loop;
 573                 case '}':
 574                     reader.scanChar(); tk = TokenKind.RBRACE; break loop;
 575                 case '/':
 576                     reader.scanChar();
 577                     if (reader.ch == '/') {
 578                         do {
 579                             reader.scanCommentChar();
 580                         } while (reader.ch != CR && reader.ch != LF && reader.bp < reader.buflen);
 581                         if (reader.bp < reader.buflen) {
 582                             comments = addComment(comments, processComment(pos, reader.bp, CommentStyle.LINE));
 583                         }
 584                         break;
 585                     } else if (reader.ch == '*') {
 586                         boolean isEmpty = false;
 587                         reader.scanChar();
 588                         CommentStyle style;
 589                         if (reader.ch == '*') {
 590                             style = CommentStyle.JAVADOC;
 591                             reader.scanCommentChar();
 592                             if (reader.ch == '/') {
 593                                 isEmpty = true;
 594                             }
 595                         } else {
 596                             style = CommentStyle.BLOCK;
 597                         }
 598                         while (!isEmpty && reader.bp < reader.buflen) {
 599                             if (reader.ch == '*') {
 600                                 reader.scanChar();
 601                                 if (reader.ch == '/') break;
 602                             } else {
 603                                 reader.scanCommentChar();
 604                             }
 605                         }
 606                         if (reader.ch == '/') {
 607                             reader.scanChar();
 608                             comments = addComment(comments, processComment(pos, reader.bp, style));
 609                             break;
 610                         } else {
 611                             lexError(pos, Errors.UnclosedComment);
 612                             break loop;
 613                         }
 614                     } else if (reader.ch == '=') {
 615                         tk = TokenKind.SLASHEQ;
 616                         reader.scanChar();
 617                     } else {
 618                         tk = TokenKind.SLASH;
 619                     }
 620                     break loop;
 621                 case '\'':
 622                     reader.scanChar();
 623                     if (reader.ch == '\'') {
 624                         lexError(pos, Errors.EmptyCharLit);
 625                         reader.scanChar();
 626                     } else {
 627                         if (reader.ch == CR || reader.ch == LF)
 628                             lexError(pos, Errors.IllegalLineEndInCharLit);
 629                         scanLitChar(pos);
 630                         if (reader.ch == '\'') {
 631                             reader.scanChar();
 632                             tk = TokenKind.CHARLITERAL;
 633                         } else {
 634                             lexError(pos, Errors.UnclosedCharLit);
 635                         }
 636                     }
 637                     break loop;
 638                 case '\"':
 639                     reader.scanChar();
 640                     while (reader.ch != '\"' && reader.ch != CR && reader.ch != LF && reader.bp < reader.buflen)
 641                         scanLitChar(pos);
 642                     if (reader.ch == '\"') {
 643                         tk = TokenKind.STRINGLITERAL;
 644                         reader.scanChar();
 645                     } else {
 646                         lexError(pos, Errors.UnclosedStrLit);
 647                     }
 648                     break loop;
 649                default:
 650                     if (isSpecial(reader.ch)) {
 651                         scanOperator();
 652                     } else {
 653                         boolean isJavaIdentifierStart;
 654                         int codePoint = -1;
 655                         if (reader.ch < '\u0080') {
 656                             // all ASCII range chars already handled, above
 657                             isJavaIdentifierStart = false;
 658                         } else {
 659                             codePoint = reader.peekSurrogates();
 660                             if (codePoint >= 0) {
 661                                 if (isJavaIdentifierStart = Character.isJavaIdentifierStart(codePoint)) {
 662                                     reader.putChar(true);
 663                                 }
 664                             } else {
 665                                 isJavaIdentifierStart = Character.isJavaIdentifierStart(reader.ch);
 666                             }
 667                         }
 668                         if (isJavaIdentifierStart) {
 669                             scanIdent();
 670                         } else if (reader.digit(pos, 10) >= 0) {
 671                             scanNumber(pos, 10);
 672                         } else if (reader.bp == reader.buflen || reader.ch == EOI && reader.bp + 1 == reader.buflen) { // JLS 3.5
 673                             tk = TokenKind.EOF;
 674                             pos = reader.realLength;
 675                         } else {
 676                             String arg;
 677 
 678                             if (codePoint >= 0) {
 679                                 char high = reader.ch;
 680                                 reader.scanChar();
 681                                 arg = String.format("\\u%04x\\u%04x", (int) high, (int)reader.ch);
 682                             } else {
 683                                 arg = (32 < reader.ch && reader.ch < 127) ?
 684                                                 String.format("%s", reader.ch) :
 685                                                 String.format("\\u%04x", (int)reader.ch);
 686                             }
 687                             lexError(pos, Errors.IllegalChar(arg));
 688                             reader.scanChar();
 689                         }
 690                     }
 691                     break loop;
 692                 }
 693             }
 694             endPos = reader.bp;
 695             switch (tk.tag) {
 696                 case DEFAULT: return new Token(tk, pos, endPos, comments);
 697                 case NAMED: return new NamedToken(tk, pos, endPos, name, comments);
 698                 case STRING: return new StringToken(tk, pos, endPos, reader.chars(), comments);
 699                 case NUMERIC: return new NumericToken(tk, pos, endPos, reader.chars(), radix, comments);
 700                 default: throw new AssertionError();
 701             }
 702         }
 703         finally {
 704             if (scannerDebug) {
 705                     System.out.println("nextToken(" + pos
 706                                        + "," + endPos + ")=|" +
 707                                        new String(reader.getRawCharacters(pos, endPos))
 708                                        + "|");
 709             }
 710         }
 711     }
 712     //where
 713         List<Comment> addComment(List<Comment> comments, Comment comment) {
 714             return comments == null ?
 715                     List.of(comment) :
 716                     comments.prepend(comment);
 717         }
 718 
 719     /** Return the position where a lexical error occurred;
 720      */
 721     public int errPos() {
 722         return errPos;
 723     }
 724 
 725     /** Set the position where a lexical error occurred;
 726      */
 727     public void errPos(int pos) {
 728         errPos = pos;
 729     }
 730 
 731     /**
 732      * Called when a complete comment has been scanned. pos and endPos
 733      * will mark the comment boundary.
 734      */
 735     protected Tokens.Comment processComment(int pos, int endPos, CommentStyle style) {
 736         if (scannerDebug)
 737             System.out.println("processComment(" + pos
 738                                + "," + endPos + "," + style + ")=|"
 739                                + new String(reader.getRawCharacters(pos, endPos))
 740                                + "|");
 741         char[] buf = reader.getRawCharacters(pos, endPos);
 742         return new BasicComment<>(new UnicodeReader(fac, buf, buf.length), style);
 743     }
 744 
 745     /**
 746      * Called when a complete whitespace run has been scanned. pos and endPos
 747      * will mark the whitespace boundary.
 748      */
 749     protected void processWhiteSpace(int pos, int endPos) {
 750         if (scannerDebug)
 751             System.out.println("processWhitespace(" + pos
 752                                + "," + endPos + ")=|" +
 753                                new String(reader.getRawCharacters(pos, endPos))
 754                                + "|");
 755     }
 756 
 757     /**
 758      * Called when a line terminator has been processed.
 759      */
 760     protected void processLineTerminator(int pos, int endPos) {
 761         if (scannerDebug)
 762             System.out.println("processTerminator(" + pos
 763                                + "," + endPos + ")=|" +
 764                                new String(reader.getRawCharacters(pos, endPos))
 765                                + "|");
 766     }
 767 
 768     /** Build a map for translating between line numbers and
 769      * positions in the input.
 770      *
 771      * @return a LineMap */
 772     public Position.LineMap getLineMap() {
 773         return Position.makeLineMap(reader.getRawCharacters(), reader.buflen, false);
 774     }
 775 
 776 
 777     /**
 778     * Scan a documentation comment; determine if a deprecated tag is present.
 779     * Called once the initial /, * have been skipped, positioned at the second *
 780     * (which is treated as the beginning of the first line).
 781     * Stops positioned at the closing '/'.
 782     */
 783     protected static class BasicComment<U extends UnicodeReader> implements Comment {
 784 
 785         CommentStyle cs;
 786         U comment_reader;
 787 
 788         protected boolean deprecatedFlag = false;
 789         protected boolean scanned = false;
 790 
 791         protected BasicComment(U comment_reader, CommentStyle cs) {
 792             this.comment_reader = comment_reader;
 793             this.cs = cs;
 794         }
 795 
 796         public String getText() {
 797             return null;
 798         }
 799 
 800         public int getSourcePos(int pos) {
 801             return -1;
 802         }
 803 
 804         public CommentStyle getStyle() {
 805             return cs;
 806         }
 807 
 808         public boolean isDeprecated() {
 809             if (!scanned && cs == CommentStyle.JAVADOC) {
 810                 scanDocComment();
 811             }
 812             return deprecatedFlag;
 813         }
 814 
 815         @SuppressWarnings("fallthrough")
 816         protected void scanDocComment() {
 817             try {
 818                 boolean deprecatedPrefix = false;
 819 
 820                 comment_reader.bp += 3; // '/**'
 821                 comment_reader.ch = comment_reader.buf[comment_reader.bp];
 822 
 823                 forEachLine:
 824                 while (comment_reader.bp < comment_reader.buflen) {
 825 
 826                     // Skip optional WhiteSpace at beginning of line
 827                     while (comment_reader.bp < comment_reader.buflen && (comment_reader.ch == ' ' || comment_reader.ch == '\t' || comment_reader.ch == FF)) {
 828                         comment_reader.scanCommentChar();
 829                     }
 830 
 831                     // Skip optional consecutive Stars
 832                     while (comment_reader.bp < comment_reader.buflen && comment_reader.ch == '*') {
 833                         comment_reader.scanCommentChar();
 834                         if (comment_reader.ch == '/') {
 835                             return;
 836                         }
 837                     }
 838 
 839                     // Skip optional WhiteSpace after Stars
 840                     while (comment_reader.bp < comment_reader.buflen && (comment_reader.ch == ' ' || comment_reader.ch == '\t' || comment_reader.ch == FF)) {
 841                         comment_reader.scanCommentChar();
 842                     }
 843 
 844                     deprecatedPrefix = false;
 845                     // At beginning of line in the JavaDoc sense.
 846                     if (!deprecatedFlag) {
 847                         String deprecated = "@deprecated";
 848                         int i = 0;
 849                         while (comment_reader.bp < comment_reader.buflen && comment_reader.ch == deprecated.charAt(i)) {
 850                             comment_reader.scanCommentChar();
 851                             i++;
 852                             if (i == deprecated.length()) {
 853                                 deprecatedPrefix = true;
 854                                 break;
 855                             }
 856                         }
 857                     }
 858 
 859                     if (deprecatedPrefix && comment_reader.bp < comment_reader.buflen) {
 860                         if (Character.isWhitespace(comment_reader.ch)) {
 861                             deprecatedFlag = true;
 862                         } else if (comment_reader.ch == '*') {
 863                             comment_reader.scanCommentChar();
 864                             if (comment_reader.ch == '/') {
 865                                 deprecatedFlag = true;
 866                                 return;
 867                             }
 868                         }
 869                     }
 870 
 871                     // Skip rest of line
 872                     while (comment_reader.bp < comment_reader.buflen) {
 873                         switch (comment_reader.ch) {
 874                             case '*':
 875                                 comment_reader.scanCommentChar();
 876                                 if (comment_reader.ch == '/') {
 877                                     return;
 878                                 }
 879                                 break;
 880                             case CR: // (Spec 3.4)
 881                                 comment_reader.scanCommentChar();
 882                                 if (comment_reader.ch != LF) {
 883                                     continue forEachLine;
 884                                 }
 885                             /* fall through to LF case */
 886                             case LF: // (Spec 3.4)
 887                                 comment_reader.scanCommentChar();
 888                                 continue forEachLine;
 889                             default:
 890                                 comment_reader.scanCommentChar();
 891                         }
 892                     } // rest of line
 893                 } // forEachLine
 894                 return;
 895             } finally {
 896                 scanned = true;
 897             }
 898         }
 899     }
 900 }