/** * Returns the length of the matched text region. */ public final int yylength() { return input.readLength(); }
break; case GroovyTokenTypes.EOF: if (lexerInput.readLength() > 0) { return recovery();
/** * Returns the length of the matched text region. */ public final int yylength() { return input.readLength(); }
if (input.readLength() > 0) { return token(RubyTokenId.IDENTIFIER, input.readLength()); // XXX? } else { return null; if (token == -1) { // EOF if (input.readLength() > 0) { return token(RubyTokenId.IDENTIFIER, input.readLength()); // XXX? } else { return null;
/** * Returns the length of the matched text region. */ public final int yylength() { return input.readLength(); }
private Token<RhtmlTokenId> token(RhtmlTokenId id) { if(input.readLength() == 0) { new Exception("Error - token length is zero!; state = " + state).printStackTrace(); } Token<RhtmlTokenId> t = tokenFactory.createToken(id); return t; }
private Token<ELTokenId> token(ELTokenId tokenId) { if(LOG) { if(input.readLength() == 0) { LOGGER.log(Level.INFO, "[" + this.getClass().getSimpleName() + "] Found zero length token: "); //NOI18N } LOGGER.log(Level.INFO, "[" + this.getClass().getSimpleName() + "] token ('" + input.readText().toString() + "'; id=" + tokenId + ")\n"); //NOI18N } return tokenFactory.createToken(tokenId); }
private void checkToken(JspTokenId tokenId) { if(input.readLength() == 0) { LOGGER.log(Level.INFO, "Found zero length token: "); } LOGGER.log(Level.INFO, "[" + this.getClass().getSimpleName() + "] token ('" + input.readText().toString() + "'; id=" + tokenId + "; state=" + state() + ")\n"); }
@Override public Token<CPTokenId> nextToken() { //just read whole input while(input.read() != LexerInput.EOF) {}; //and create one big token return input.readLength() > 0 ? tokenFactory.createToken(CPTokenId.CSS) : null; }
private Token<GroovyTokenId> recovery() { int len = lexerInput.readLength() - myCharBuffer.getExtraCharCount(); int tokenLength = lexerInput.readLength(); scanner.resetText(); while (len < tokenLength) { LOG.finest("Consuming character"); scannerConsumeChar(); len++; } return tokenLength > 0 ? createToken(GroovyTokenId.ERROR.ordinal(), tokenLength) : null; }
private CharSequence getPossibleTagName() { int actChar; int prev_read = input.readLength(); //remember the size of the read sequence int read = 0; while(true) { actChar = input.read(); read++; if(!(Character.isLetter(actChar) || Character.isDigit(actChar) || (actChar == '_') || (actChar == '-') || (actChar == ':') || (actChar == '.') || (actChar == '/')) || (actChar == EOF)) { // EOL or not alpha //end of tagname CharSequence tagName = input.readText().subSequence(prev_read, prev_read + read - 1); input.backup(read); //put the lookahead text back to the buffer return tagName; } } }
@NonNull private Token<DockerfileTokenId> token(DockerfileTokenId id) { final String fixedText = id.fixedText(); return (fixedText != null && fixedText.length() == input.readLength()) ? tokenFactory.getFlyweightToken(id, fixedText) : tokenFactory.createToken(id); }
private Token<JspTokenId> scriptletToken(JspTokenId tokenId, int javaCodeType) { if(LOG) { checkToken(tokenId); } JspTokenId.JavaCodeType scriptletType; switch(javaCodeType) { case JAVA_SCRITPLET: case JAVA_SCRITPLET_DOCUMENT: scriptletType = JspTokenId.JavaCodeType.SCRIPTLET; break; case JAVA_DECLARATION: case JAVA_DECLARATION_DOCUMENT: scriptletType = JspTokenId.JavaCodeType.DECLARATION; break; case JAVA_EXPRESSION: case JAVA_EXPRESSION_DOCUMENT: scriptletType = JspTokenId.JavaCodeType.EXPRESSION; break; default: throw new IllegalStateException("Unsupported scriptlet type " + lexerStateJspScriptlet); } return tokenFactory.createPropertyToken(tokenId, input.readLength(), new JspTokenPropertyProvider(scriptletType), PartType.COMPLETE); }
/** * Creates token */ private Token<FortranTokenId> token(FortranTokenId id, String fixedText, PartType part) { assert id != null : "id must be not null"; Token<FortranTokenId> token; if (fixedText != null) { // create flyweight token token = tokenFactory.getFlyweightToken(id, fixedText); } else { if (part != PartType.COMPLETE) { token = tokenFactory.createToken(id, input.readLength(), part); } else { token = tokenFactory.createToken(id); } } assert token != null : "token must be created as result for " + id; return token; }
private Token<CppStringTokenId> token(CppStringTokenId id, String fixedText, PartType part) { assert id != null : "id must be not null"; Token<CppStringTokenId> token; if (fixedText != null && !escapedLF) { // create flyweight token token = tokenFactory.getFlyweightToken(id, fixedText); } else { if (part != PartType.COMPLETE) { token = tokenFactory.createToken(id, input.readLength(), part); } else { token = tokenFactory.createToken(id); } } escapedLF = false; assert token != null : "token must be created as result for " + id; return token; }
case EOF: if (input.readLength() > 0) { return token(RubyStringTokenId.STRING_TEXT); } else { if (input.readLength() > 1) { // already read some text input.backup(1); input.readLength());
private Token<CppTokenId> token(CppTokenId id, String fixedText, PartType part) { assert id != null : "id must be not null"; Token<CppTokenId> token; if (fixedText != null && !isTokenSplittedByEscapedLine()) { // create flyweight token token = tokenFactory.getFlyweightToken(id, fixedText); } else { if (part != PartType.COMPLETE) { token = tokenFactory.createToken(id, input.readLength(), part); } else { token = tokenFactory.createToken(id); } } tokenSplittedByEscapedLine = 0; escapedEatenChars = 0; assert token != null : "token must be created as result for " + id; postTokenCreate(id); return token; }
return tokenFactory.createToken( DockerfileTokenId.STRING_LITERAL, input.readLength(), PartType.START); } else if (c == '\n') { //NOI18N return tokenFactory.createToken( DockerfileTokenId.STRING_LITERAL, input.readLength(), PartType.START); } else if (c == EOF) { return tokenFactory.createToken( DockerfileTokenId.STRING_LITERAL, input.readLength(), PartType.START); } else if (c == closingChar) {
if ((wasFirst && !CndLexerUtilities.isCppIdentifierStart(ch)) || (!CndLexerUtilities.isCppIdentifierPart(ch))) { input.backup(1); if (input.readLength() > 1) { return tokenFactory.createToken(DoxygenTokenId.TAG, input.readLength()); } else { return tokenFactory.createToken(DoxygenTokenId.OTHER_TEXT, input.readLength());
if (c == EOF || !Character.isWhitespace(c)) { // Return single space as flyweight token backup(1); return input.readLength() == 1 ? tokenFactory.getFlyweightToken(DockerfileTokenId.WHITESPACE, " ") //NOI18N : tokenFactory.createToken(DockerfileTokenId.WHITESPACE);