private int getIdentifier(int first) throws IOException { if (isMultiByteChar(first)) first = src.readCodepoint(first, encoding); if (!isIdentifierChar(first)) return first; tokenBuffer.append((char) first); int c; for (c = src.read(); c != EOF; c = src.read()) { if (isMultiByteChar(c)) c = src.readCodepoint(c, encoding); if (!isIdentifierChar(c)) break; tokenBuffer.append((char) c); } src.unread(c); return first; }
private int getIdentifier(int first) throws IOException { if (isMultiByteChar(first)) first = src.readCodepoint(first, encoding); if (!isIdentifierChar(first)) return first; tokenBuffer.append((char) first); int c; for (c = src.read(); c != EOF; c = src.read()) { if (isMultiByteChar(c)) c = src.readCodepoint(c, encoding); if (!isIdentifierChar(c)) break; tokenBuffer.append((char) c); } src.unread(c); return first; }
private int addNonAsciiToBuffer(int c, LexerSource src, Encoding encoding, RubyYaccLexer lexer, ByteList buffer) throws SyntaxException, IOException { c = src.readCodepoint(c, encoding); if (c == -2) { // FIXME: Hack throw new SyntaxException(PID.INVALID_MULTIBYTE_CHAR, lexer.getPosition(), null, "invalid multibyte char (" + encoding + ")"); } // FIXME: We basically go from bytes to codepoint back to bytes to append them...fix this return lexer.tokenAddMBC(c, buffer); } }
private int addNonAsciiToBuffer(int c, LexerSource src, Encoding encoding, RubyYaccLexer lexer, ByteList buffer) throws SyntaxException, IOException { c = src.readCodepoint(c, encoding); if (c == -2) { // FIXME: Hack throw new SyntaxException(PID.INVALID_MULTIBYTE_CHAR, lexer.getPosition(), null, "invalid multibyte char (" + encoding + ")"); } // FIXME: We basically go from bytes to codepoint back to bytes to append them...fix this return lexer.tokenAddMBC(c, buffer); } }