private boolean isNext_identchar() throws IOException { int c = src.read(); src.unread(c); return c != EOF && (Character.isLetterOrDigit(c) || c == '_'); }
private int getIdentifier(int first) throws IOException { if (isMultiByteChar(first)) first = src.readCodepoint(first, encoding); if (!isIdentifierChar(first)) return first; tokenBuffer.append((char) first); int c; for (c = src.read(); c != EOF; c = src.read()) { if (isMultiByteChar(c)) c = src.readCodepoint(c, encoding); if (!isIdentifierChar(c)) break; tokenBuffer.append((char) c); } src.unread(c); return first; }
private boolean isNext_identchar() throws IOException { int c = src.read(); src.unread(c); return c != EOF && (Character.isLetterOrDigit(c) || c == '_'); }
private int getIdentifier(int first) throws IOException { if (isMultiByteChar(first)) first = src.readCodepoint(first, encoding); if (!isIdentifierChar(first)) return first; tokenBuffer.append((char) first); int c; for (c = src.read(); c != EOF; c = src.read()) { if (isMultiByteChar(c)) c = src.readCodepoint(c, encoding); if (!isIdentifierChar(c)) break; tokenBuffer.append((char) c); } src.unread(c); return first; }
private char scanOct(int count) throws IOException { char value = '\0'; for (int i = 0; i < count; i++) { int c = src.read(); if (!isOctChar(c)) { src.unread(c); break; } value <<= 3; value |= Integer.parseInt("" + (char) c, 8); } return value; } }
private char scanOct(int count) throws IOException { char value = '\0'; for (int i = 0; i < count; i++) { int c = src.read(); if (!isOctChar(c)) { src.unread(c); break; } value <<= 3; value |= Integer.parseInt("" + (char) c, 8); } return value; } }
private void detectUTF8BOM() throws IOException { int b1 = src.read(); if (b1 == 0xef) { int b2 = src.read(); if (b2 == 0xbb) { int b3 = src.read(); if (b3 == 0xbf) { setEncoding(UTF8_ENCODING); } else { src.unread(b3); src.unread(b2); src.unread(b1); } } else { src.unread(b2); src.unread(b1); } } else { src.unread(b1); } }
public int readCodepoint(int first, Encoding encoding) throws IOException { int count = 0; byte[] value = new byte[6]; // We know this will never be EOF value[0] = (byte) first; for (count = 1; count < 6; count++) { int c = read(); if (c == RubyYaccLexer.EOF) break; // Maybe we have enough bytes read to mbc at EOF. value[count] = (byte) c; } int length = encoding.length(value, 0, count); if (length < 0) { return -2; // TODO: Hack } int codepoint = encoding.mbcToCode(value, 0, length); for (int i = count - 1; i >= length; i--) { unread(value[i]); } return codepoint; }
public void tokenAddMBCFromSrc(int c, ByteList buffer) throws IOException { // read bytes for length of character int length = buffer.getEncoding().length((byte)c); buffer.append((byte)c); for (int off = 0; off < length - 1; off++) { buffer.append((byte)src.read()); } }
public void tokenAddMBCFromSrc(int c, ByteList buffer) throws IOException { // read bytes for length of character int length = buffer.getEncoding().length((byte)c); buffer.append((byte)c); for (int off = 0; off < length - 1; off++) { buffer.append((byte)src.read()); } }
private int tilde() throws IOException { int c; if (lex_state == LexState.EXPR_FNAME || lex_state == LexState.EXPR_DOT) { if ((c = src.read()) != '@') src.unread(c); setState(LexState.EXPR_ARG); } else { setState(LexState.EXPR_BEG); } yaccValue = new Token("~", getPosition()); return Tokens.tTILDE; }
private int tilde() throws IOException { int c; if (lex_state == LexState.EXPR_FNAME || lex_state == LexState.EXPR_DOT) { if ((c = src.read()) != '@') src.unread(c); setState(LexState.EXPR_ARG); } else { setState(LexState.EXPR_BEG); } yaccValue = new Token("~", getPosition()); return Tokens.tTILDE; }
private void escaped(RubyYaccLexer lexer, Encoding encoding, LexerSource src, ByteList buffer) throws java.io.IOException { int c; switch (c = src.read()) { case '\\': parseEscapeIntoBuffer(lexer, encoding, src, buffer); break; case RubyYaccLexer.EOF: throw new SyntaxException(PID.INVALID_ESCAPE_SYNTAX, src.getPosition(), src.getCurrentLine(), "Invalid escape character syntax"); default: buffer.append(c); } }
private void escaped(RubyYaccLexer lexer, Encoding encoding, LexerSource src, ByteList buffer) throws java.io.IOException { int c; switch (c = src.read()) { case '\\': parseEscapeIntoBuffer(lexer, encoding, src, buffer); break; case RubyYaccLexer.EOF: throw new SyntaxException(PID.INVALID_ESCAPE_SYNTAX, src.getPosition(), src.getCurrentLine(), "Invalid escape character syntax"); default: buffer.append(c); } }
private int percent(boolean spaceSeen) throws IOException { if (isBEG()) return parseQuote(src.read()); int c = src.read(); if (c == '=') { setState(LexState.EXPR_BEG); yaccValue = new Token("%", getPosition()); return Tokens.tOP_ASGN; } if (isSpaceArg(c, spaceSeen)) return parseQuote(c); determineExpressionState(); src.unread(c); yaccValue = new Token("%", getPosition()); warn_balanced(c, spaceSeen, "%%", "string literal"); return Tokens.tPERCENT; }
private int caret() throws IOException { int c = src.read(); if (c == '=') { setState(LexState.EXPR_BEG); yaccValue = new Token("^", getPosition()); return Tokens.tOP_ASGN; } determineExpressionState(); src.unread(c); yaccValue = new Token("^", getPosition()); return Tokens.tCARET; }
private int percent(boolean spaceSeen) throws IOException { if (isBEG()) return parseQuote(src.read()); int c = src.read(); if (c == '=') { setState(LexState.EXPR_BEG); yaccValue = new Token("%", getPosition()); return Tokens.tOP_ASGN; } if (isSpaceArg(c, spaceSeen)) return parseQuote(c); determineExpressionState(); src.unread(c); yaccValue = new Token("%", getPosition()); warn_balanced(c, spaceSeen, "%%", "string literal"); return Tokens.tPERCENT; }
private int dot() throws IOException { int c; setState(LexState.EXPR_BEG); if ((c = src.read()) == '.') { if ((c = src.read()) == '.') { yaccValue = new Token("...", getPosition()); return Tokens.tDOT3; } src.unread(c); yaccValue = new Token("..", getPosition()); return Tokens.tDOT2; } src.unread(c); if (Character.isDigit(c)) { throw new SyntaxException(PID.FLOAT_MISSING_ZERO, getPosition(), getCurrentLine(), "no .<digit> floating literal anymore; put 0 before dot"); } setState(LexState.EXPR_DOT); yaccValue = new Token(".", getPosition()); return Tokens.tDOT; }
private int caret() throws IOException { int c = src.read(); if (c == '=') { setState(LexState.EXPR_BEG); yaccValue = new Token("^", getPosition()); return Tokens.tOP_ASGN; } determineExpressionState(); src.unread(c); yaccValue = new Token("^", getPosition()); return Tokens.tCARET; }
private int greaterThan() throws IOException { determineExpressionState(); int c = src.read(); switch (c) { case '=': yaccValue = new Token(">=", getPosition()); return Tokens.tGEQ; case '>': if ((c = src.read()) == '=') { setState(LexState.EXPR_BEG); yaccValue = new Token(">>", getPosition()); return Tokens.tOP_ASGN; } src.unread(c); yaccValue = new Token(">>", getPosition()); return Tokens.tRSHFT; default: src.unread(c); yaccValue = new Token(">", getPosition()); return Tokens.tGT; } }