@Override protected Comment processComment(int pos, int endPos, CommentStyle style) { char[] buf = reader.getRawCharacters(pos, endPos); return new CommentWithTextAndPosition( pos, endPos, new AccessibleReader(fac, buf, buf.length), style); } }
if (reader.peekChar() == '\\' && !reader.isUnicode()) { reader.skipChar(); reader.putChar('\\', true); } else { reader.scanChar(); switch (reader.ch) { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': char leadch = reader.ch; int oct = reader.digit(pos, 8); reader.scanChar(); if ('0' <= reader.ch && reader.ch <= '7') { oct = oct * 8 + reader.digit(pos, 8); reader.scanChar(); if (leadch <= '3' && '0' <= reader.ch && reader.ch <= '7') { oct = oct * 8 + reader.digit(pos, 8); reader.scanChar(); reader.putChar((char)oct); break; case 'b': reader.putChar('\b', true); break; case 't': reader.putChar('\t', true); break; case 'n': reader.putChar('\n', true); break; case 'f': reader.putChar('\f', true); break; case 'r':
case FF: // (Spec 3.6) do { reader.scanChar(); } while (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF); processWhiteSpace(pos, reader.bp); break; case LF: // (Spec 3.4) reader.scanChar(); processLineTerminator(pos, reader.bp); break; case CR: // (Spec 3.4) reader.scanChar(); if (reader.ch == LF) { reader.scanChar(); break loop; case '0': reader.scanChar(); if (reader.ch == 'x' || reader.ch == 'X') { reader.scanChar(); skipIllegalUnderscores(); if (reader.ch == '.') { scanHexFractionAndSuffix(pos, false); } else if (reader.digit(pos, 16) < 0) { lexError(pos, "invalid.hex.number"); } else { reader.scanChar(); skipIllegalUnderscores();
/** Read next character in comment, skipping over double '\' characters. */ protected void scanCommentChar() { scanChar(); if (ch == '\\') { if (peekChar() == '\\' && !isUnicode()) { skipChar(); } else { convertUnicode(); } } }
private void scanDigits(int pos, int digitRadix) { char saveCh; int savePos; do { if (reader.ch != '_') { reader.putChar(false); } else { if (!allowUnderscoresInLiterals) { lexError(pos, "unsupported.underscore.lit", source.name); allowUnderscoresInLiterals = true; } } saveCh = reader.ch; savePos = reader.bp; reader.scanChar(); } while (reader.digit(pos, digitRadix) >= 0 || reader.ch == '_'); if (saveCh == '_') lexError(savePos, "illegal.underscore"); }
boolean isJavaIdentifierPart; char high; reader.putChar(true); do { switch (reader.ch) { case '\u0018': case '\u0019': case '\u001B': case '\u007F': reader.scanChar(); continue; case '\u001A': // EOI is also a legal identifier part if (reader.bp >= reader.buflen) { name = reader.name(); tk = tokens.lookupKind(name); return; reader.scanChar(); continue; default: } else { if (Character.isIdentifierIgnorable(reader.ch)) { reader.scanChar(); continue; } else { high = reader.scanSurrogates(); if (high != 0) { reader.putChar(high); isJavaIdentifierPart = Character.isJavaIdentifierPart(
/** * Called when a complete comment has been scanned. pos and endPos * will mark the comment boundary. */ protected Tokens.Comment processComment(int pos, int endPos, CommentStyle style) { if (scannerDebug) System.out.println("processComment(" + pos + "," + endPos + "," + style + ")=|" + new String(reader.getRawCharacters(pos, endPos)) + "|"); char[] buf = reader.getRawCharacters(pos, endPos); return new BasicComment<UnicodeReader>(new UnicodeReader(fac, buf, buf.length), style); }
/** Read longest possible sequence of special characters and convert * to token. */ private void scanOperator() { while (true) { reader.putChar(false); Name newname = reader.name(); TokenKind tk1 = tokens.lookupKind(newname); if (tk1 == TokenKind.IDENTIFIER) { reader.sp--; break; } tk = tk1; reader.scanChar(); if (!isSpecial(reader.ch)) break; } }
protected void putChar(boolean scan) { putChar(ch, scan); }
protected UnicodeReader(ScannerFactory sf, char[] input, int inputLength) { log = sf.log; names = sf.names; if (inputLength == input.length) { if (input.length > 0 && Character.isWhitespace(input[input.length - 1])) { inputLength--; } else { input = Arrays.copyOf(input, inputLength + 1); } } buf = input; buflen = inputLength; buf[buflen] = EOI; bp = -1; scanChar(); }
/** Read fractional part and 'd' or 'f' suffix of floating point number. */ private void scanHexFractionAndSuffix(int pos, boolean seendigit) { radix = 16; Assert.check(reader.ch == '.'); reader.putChar(true); skipIllegalUnderscores(); if (reader.digit(pos, 16) >= 0) { seendigit = true; scanDigits(pos, 16); } if (!seendigit) lexError(pos, "invalid.hex.number"); else scanHexExponentAndSuffix(pos); }
/** * Create a scanner from the input array. This method might * modify the array. To avoid copying the input array, ensure * that {@code inputLength < input.length} or * {@code input[input.length -1]} is a white space character. * * @param fac the factory which created this Scanner * @param buf the input, might be modified * Must be positive and less than or equal to input.length. */ protected JavaTokenizer(ScannerFactory fac, CharBuffer buf) { this(fac, new UnicodeReader(fac, buf)); }
/** Read next character. */ protected void scanChar() { if (bp < buflen) { ch = buf[++bp]; if (ch == '\\') { convertUnicode(); } } }
if (reader.peekChar() == '\\' && !reader.isUnicode()) { reader.skipChar(); reader.putChar('\\', true); } else { reader.scanChar(); switch (reader.ch) { case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': char leadch = reader.ch; int oct = reader.digit(pos, 8); reader.scanChar(); if ('0' <= reader.ch && reader.ch <= '7') { oct = oct * 8 + reader.digit(pos, 8); reader.scanChar(); if (leadch <= '3' && '0' <= reader.ch && reader.ch <= '7') { oct = oct * 8 + reader.digit(pos, 8); reader.scanChar(); reader.putChar((char)oct); break; case 'b': reader.putChar('\b', true); break; case 't': reader.putChar('\t', true); break; case 'n': reader.putChar('\n', true); break; case 'f': reader.putChar('\f', true); break; case 'r':
case FF: // (Spec 3.6) do { reader.scanChar(); } while (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF); processWhiteSpace(pos, reader.bp); break; case LF: // (Spec 3.4) reader.scanChar(); processLineTerminator(pos, reader.bp); break; case CR: // (Spec 3.4) reader.scanChar(); if (reader.ch == LF) { reader.scanChar(); break loop; case '0': reader.scanChar(); if (reader.ch == 'x' || reader.ch == 'X') { reader.scanChar(); skipIllegalUnderscores(); if (reader.ch == '.') { scanHexFractionAndSuffix(pos, false); } else if (reader.digit(pos, 16) < 0) { lexError(pos, "invalid.hex.number"); } else { reader.scanChar(); skipIllegalUnderscores();
/** Read next character in comment, skipping over double '\' characters. */ protected void scanCommentChar() { scanChar(); if (ch == '\\') { if (peekChar() == '\\' && !isUnicode()) { skipChar(); } else { convertUnicode(); } } }
private void scanDigits(int pos, int digitRadix) { char saveCh; int savePos; do { if (reader.ch != '_') { reader.putChar(false); } else { if (!allowUnderscoresInLiterals) { lexError(pos, "unsupported.underscore.lit", source.name); allowUnderscoresInLiterals = true; } } saveCh = reader.ch; savePos = reader.bp; reader.scanChar(); } while (reader.digit(pos, digitRadix) >= 0 || reader.ch == '_'); if (saveCh == '_') lexError(savePos, "illegal.underscore"); }
boolean isJavaIdentifierPart; char high; reader.putChar(true); do { switch (reader.ch) { case '\u0018': case '\u0019': case '\u001B': case '\u007F': reader.scanChar(); continue; case '\u001A': // EOI is also a legal identifier part if (reader.bp >= reader.buflen) { name = reader.name(); tk = tokens.lookupKind(name); return; reader.scanChar(); continue; default: } else { if (Character.isIdentifierIgnorable(reader.ch)) { reader.scanChar(); continue; } else { high = reader.scanSurrogates(); if (high != 0) { reader.putChar(high); isJavaIdentifierPart = Character.isJavaIdentifierPart(
/** * Called when a complete comment has been scanned. pos and endPos * will mark the comment boundary. */ protected Tokens.Comment processComment(int pos, int endPos, CommentStyle style) { if (scannerDebug) System.out.println("processComment(" + pos + "," + endPos + "," + style + ")=|" + new String(reader.getRawCharacters(pos, endPos)) + "|"); char[] buf = reader.getRawCharacters(pos, endPos); return new BasicComment<UnicodeReader>(new UnicodeReader(fac, buf, buf.length), style); }