/** * Create a scanner from the input array. This method might * modify the array. To avoid copying the input array, ensure * that {@code inputLength < input.length} or * {@code input[input.length -1]} is a white space character. * * @param fac the factory which created this Scanner * @param buf the input, might be modified * Must be positive and less than or equal to input.length. */ protected Scanner(ScannerFactory fac, CharBuffer buf) { this(fac, new JavaTokenizer(fac, buf)); }
public void errPos(int pos) { tokenizer.errPos(pos); } }
reader.scanChar(); } while (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF); processWhiteSpace(pos, reader.bp); break; case LF: // (Spec 3.4) reader.scanChar(); processLineTerminator(pos, reader.bp); break; case CR: // (Spec 3.4) processLineTerminator(pos, reader.bp); break; case 'A': case 'B': case 'C': case 'D': case 'E': case 'z': case '$': case '_': scanIdent(); break loop; case '0': if (reader.ch == 'x' || reader.ch == 'X') { reader.scanChar(); skipIllegalUnderscores(); if (reader.ch == '.') { scanHexFractionAndSuffix(pos, false); } else if (reader.digit(pos, 16) < 0) { lexError(pos, "invalid.hex.number"); } else { scanNumber(pos, 16);
/** Read fractional part of floating point number. */ private void scanFraction(int pos) { skipIllegalUnderscores(); if ('0' <= reader.ch && reader.ch <= '9') { scanDigits(pos, 10); } int sp1 = reader.sp; if (reader.ch == 'e' || reader.ch == 'E') { reader.putChar(true); skipIllegalUnderscores(); if (reader.ch == '+' || reader.ch == '-') { reader.putChar(true); } skipIllegalUnderscores(); if ('0' <= reader.ch && reader.ch <= '9') { scanDigits(pos, 10); return; } lexError(pos, "malformed.fp.lit"); reader.sp = sp1; } }
/** Read fractional part and 'd' or 'f' suffix of floating point number. */ private void scanHexFractionAndSuffix(int pos, boolean seendigit) { radix = 16; Assert.check(reader.ch == '.'); reader.putChar(true); skipIllegalUnderscores(); if (reader.digit(pos, 16) >= 0) { seendigit = true; scanDigits(pos, 16); } if (!seendigit) lexError(pos, "invalid.hex.number"); else scanHexExponentAndSuffix(pos); }
private void skipIllegalUnderscores() { if (reader.ch == '_') { lexError(reader.bp, "illegal.underscore"); while (reader.ch == '_') reader.scanChar(); } }
@Override protected Comment processComment(int pos, int endPos, CommentStyle style) { int prevEndPos = Math.max(prevEndPosition, endComment); endComment = endPos; String content = new String(reader.getRawCharacters(pos, endPos)); StartConnection start = determineStartConnection(prevEndPos, pos); EndConnection end = determineEndConnection(endPos); CommentInfo comment = new CommentInfo(prevEndPos, pos, endPos, content, start, end); comments.append(comment); return super.processComment(pos, endPos, style); }
public LineMap getLineMap() { return tokenizer.getLineMap(); }
/** Read longest possible sequence of special characters and convert * to token. */ private void scanOperator() { while (true) { reader.putChar(false); Name newname = reader.name(); TokenKind tk1 = tokens.lookupKind(newname); if (tk1 == TokenKind.IDENTIFIER) { reader.sp--; break; } tk = tk1; reader.scanChar(); if (!isSpecial(reader.ch)) break; } }
reader.scanChar(); } while (reader.ch == ' ' || reader.ch == '\t' || reader.ch == FF); processWhiteSpace(pos, reader.bp); break; case LF: // (Spec 3.4) reader.scanChar(); processLineTerminator(pos, reader.bp); break; case CR: // (Spec 3.4) processLineTerminator(pos, reader.bp); break; case 'A': case 'B': case 'C': case 'D': case 'E': case 'z': case '$': case '_': scanIdent(); break loop; case '0': if (reader.ch == 'x' || reader.ch == 'X') { reader.scanChar(); skipIllegalUnderscores(); if (reader.ch == '.') { scanHexFractionAndSuffix(pos, false); } else if (reader.digit(pos, 16) < 0) { lexError(pos, "invalid.hex.number"); } else { scanNumber(pos, 16);
/** Read fractional part of floating point number. */ private void scanFraction(int pos) { skipIllegalUnderscores(); if ('0' <= reader.ch && reader.ch <= '9') { scanDigits(pos, 10); } int sp1 = reader.sp; if (reader.ch == 'e' || reader.ch == 'E') { reader.putChar(true); skipIllegalUnderscores(); if (reader.ch == '+' || reader.ch == '-') { reader.putChar(true); } skipIllegalUnderscores(); if ('0' <= reader.ch && reader.ch <= '9') { scanDigits(pos, 10); return; } lexError(pos, "malformed.fp.lit"); reader.sp = sp1; } }
/** Read fractional part and 'd' or 'f' suffix of floating point number. */ private void scanHexFractionAndSuffix(int pos, boolean seendigit) { radix = 16; Assert.check(reader.ch == '.'); reader.putChar(true); skipIllegalUnderscores(); if (reader.digit(pos, 16) >= 0) { seendigit = true; scanDigits(pos, 16); } if (!seendigit) lexError(pos, "invalid.hex.number"); else scanHexExponentAndSuffix(pos); }
private void skipIllegalUnderscores() { if (reader.ch == '_') { lexError(reader.bp, "illegal.underscore"); while (reader.ch == '_') reader.scanChar(); } }
public LineMap getLineMap() { return tokenizer.getLineMap(); }
/** Read longest possible sequence of special characters and convert * to token. */ private void scanOperator() { while (true) { reader.putChar(false); Name newname = reader.name(); TokenKind tk1 = tokens.lookupKind(newname); if (tk1 == TokenKind.IDENTIFIER) { reader.sp--; break; } tk = tk1; reader.scanChar(); if (!isSpecial(reader.ch)) break; } }
if (reader.ch == 'p' || reader.ch == 'P') { reader.putChar(true); skipIllegalUnderscores(); if (reader.ch == '+' || reader.ch == '-') { reader.putChar(true); skipIllegalUnderscores(); if ('0' <= reader.ch && reader.ch <= '9') { scanDigits(pos, 10); if (!allowHexFloats) { lexError(pos, "unsupported.fp.lit", source.name); allowHexFloats = true; lexError(pos, "unsupported.cross.fp.lit"); } else lexError(pos, "malformed.fp.lit"); } else { lexError(pos, "malformed.fp.lit");
public void nextToken() { prevToken = token; if (!savedTokens.isEmpty()) { token = savedTokens.remove(0); } else { token = tokenizer.readToken(); } }
private void scanDigits(int pos, int digitRadix) { char saveCh; int savePos; do { if (reader.ch != '_') { reader.putChar(false); } else { if (!allowUnderscoresInLiterals) { lexError(pos, "unsupported.underscore.lit", source.name); allowUnderscoresInLiterals = true; } } saveCh = reader.ch; savePos = reader.bp; reader.scanChar(); } while (reader.digit(pos, digitRadix) >= 0 || reader.ch == '_'); if (saveCh == '_') lexError(savePos, "illegal.underscore"); }
protected Scanner(ScannerFactory fac, char[] buf, int inputLength) { this(fac, new JavaTokenizer(fac, buf, inputLength)); }