private Token<DoxygenTokenId> token(DoxygenTokenId id) { return tokenFactory.createToken(id); }
private Token<RubyTokenId> token(RubyTokenId id, int length) { String fixedText = id.fixedText(); return (fixedText != null) ? tokenFactory.getFlyweightToken(id, fixedText) : tokenFactory.createToken(id, length); }
private Token<STokenId> createToken (Marenka marenka) { Vojta v = marenka.removeFirst (); STokenId tokenId = tokenIDToType.get (v.type); assert tokenId != null : "Unknown type " + v.type; input.setIndex (v.endOffset); if (marenka.isEmpty ()) this.state = marenka.getState (); else this.state = marenka; //S ystem.out.println("nextToken <" + v.type + "," + e (input.getString (v.startOffset, v.endOffset)) + "," + v.startOffset + "," + v.endOffset); if (v.property instanceof TokenProperties) return tokenFactory.createPropertyToken ( tokenId, v.endOffset - v.startOffset, (TokenProperties) v.property, PartType.COMPLETE ); else return tokenFactory.createPropertyToken ( tokenId, v.endOffset - v.startOffset, new TokenPropProvider(v.property), PartType.COMPLETE ); }
private Token<JspTokenId> scriptletToken(JspTokenId tokenId, int javaCodeType) { if(LOG) { checkToken(tokenId); } JspTokenId.JavaCodeType scriptletType; switch(javaCodeType) { case JAVA_SCRITPLET: case JAVA_SCRITPLET_DOCUMENT: scriptletType = JspTokenId.JavaCodeType.SCRIPTLET; break; case JAVA_DECLARATION: case JAVA_DECLARATION_DOCUMENT: scriptletType = JspTokenId.JavaCodeType.DECLARATION; break; case JAVA_EXPRESSION: case JAVA_EXPRESSION_DOCUMENT: scriptletType = JspTokenId.JavaCodeType.EXPRESSION; break; default: throw new IllegalStateException("Unsupported scriptlet type " + lexerStateJspScriptlet); } return tokenFactory.createPropertyToken(tokenId, input.readLength(), new JspTokenPropertyProvider(scriptletType), PartType.COMPLETE); }
private Token<RubyCommentTokenId> token(RubyCommentTokenId id) { return tokenFactory.createToken(id); }
private Token<CppStringTokenId> token(CppStringTokenId id, String fixedText, PartType part) { assert id != null : "id must be not null"; Token<CppStringTokenId> token; if (fixedText != null && !escapedLF) { // create flyweight token token = tokenFactory.getFlyweightToken(id, fixedText); } else { if (part != PartType.COMPLETE) { token = tokenFactory.createToken(id, input.readLength(), part); } else { token = tokenFactory.createToken(id); } } escapedLF = false; assert token != null : "token must be created as result for " + id; return token; }
private Token<RubyStringTokenId> token(RubyStringTokenId id) { return tokenFactory.createToken(id); }
/** * Creates token */ private Token<FortranTokenId> token(FortranTokenId id, String fixedText, PartType part) { assert id != null : "id must be not null"; Token<FortranTokenId> token; if (fixedText != null) { // create flyweight token token = tokenFactory.getFlyweightToken(id, fixedText); } else { if (part != PartType.COMPLETE) { token = tokenFactory.createToken(id, input.readLength(), part); } else { token = tokenFactory.createToken(id); } } assert token != null : "token must be created as result for " + id; return token; }
private Token<RhtmlTokenId> token(RhtmlTokenId id) { if(input.readLength() == 0) { new Exception("Error - token length is zero!; state = " + state).printStackTrace(); } Token<RhtmlTokenId> t = tokenFactory.createToken(id); return t; }
@NonNull private Token<DockerfileTokenId> token(DockerfileTokenId id) { final String fixedText = id.fixedText(); return (fixedText != null && fixedText.length() == input.readLength()) ? tokenFactory.getFlyweightToken(id, fixedText) : tokenFactory.createToken(id); }
private Token<JspTokenId> token(JspTokenId tokenId) { if(LOG) { checkToken(tokenId); } return tokenFactory.createToken(tokenId); }
private Token<GroovyTokenId> createToken(int tokenIntId, int tokenLength) { GroovyTokenId id = getTokenId(tokenIntId); LOG.log(Level.FINEST, "Creating token: {0}, length: {1}", new Object[]{id.name(), tokenLength}); String fixedText = id.fixedText(); return (fixedText != null) ? tokenFactory.getFlyweightToken(id, fixedText) : tokenFactory.createToken(id, tokenLength); }
@Override public Token<PHPTopTokenId> nextToken() { PHPTopTokenId tokenId = scanner.nextToken(); Token<PHPTopTokenId> token = null; if (tokenId != null) { token = tokenFactory.createToken(tokenId); } return token; }
private Token<CppTokenId> token(CppTokenId id, String fixedText, PartType part) { assert id != null : "id must be not null"; Token<CppTokenId> token; if (fixedText != null && !isTokenSplittedByEscapedLine()) { // create flyweight token token = tokenFactory.getFlyweightToken(id, fixedText); } else { if (part != PartType.COMPLETE) { token = tokenFactory.createToken(id, input.readLength(), part); } else { token = tokenFactory.createToken(id); } } tokenSplittedByEscapedLine = 0; escapedEatenChars = 0; assert token != null : "token must be created as result for " + id; postTokenCreate(id); return token; }
@Override public Token<PHPTokenId> nextToken() { try { PHPTokenId tokenId = scanner.nextToken(); Token<PHPTokenId> token = null; if (tokenId != null) { token = tokenFactory.createToken(tokenId); } return token; } catch (IOException ex) { Logger.getLogger(GSFPHPLexer.class.getName()).log(Level.SEVERE, null, ex); } return null; }
backup(1); return input.readLength() == 1 ? tokenFactory.getFlyweightToken(DockerfileTokenId.WHITESPACE, " ") //NOI18N : tokenFactory.createToken(DockerfileTokenId.WHITESPACE);
@Override public Token<PHPDocCommentTokenId> nextToken() { try { PHPDocCommentTokenId tokenId = scanner.nextToken(); Token<PHPDocCommentTokenId> token = null; if (tokenId != null) { token = tokenFactory.createToken(tokenId); } return token; } catch (IOException ex) { Logger.getLogger(GSFPHPLexer.class.getName()).log(Level.SEVERE, null, ex); } return null; }
@Override public Token<CfgPropsTokenId> nextToken() { try { CfgPropsTokenId tokenId = scanner.nextTokenId(); Token<CfgPropsTokenId> token = null; if (tokenId != null) { token = tokenFactory.createToken(tokenId); } return token; } catch (IOException ex) { Logger.getLogger(CfgPropsLexer.class.getName()).log(Level.SEVERE, null, ex); } return null; }
private Token<ELTokenId> token(ELTokenId tokenId) { if(LOG) { if(input.readLength() == 0) { LOGGER.log(Level.INFO, "[" + this.getClass().getSimpleName() + "] Found zero length token: "); //NOI18N } LOGGER.log(Level.INFO, "[" + this.getClass().getSimpleName() + "] token ('" + input.readText().toString() + "'; id=" + tokenId + ")\n"); //NOI18N } return tokenFactory.createToken(tokenId); }
@Override public Token<CPTokenId> nextToken() { //just read whole input while(input.read() != LexerInput.EOF) {}; //and create one big token return input.readLength() > 0 ? tokenFactory.createToken(CPTokenId.CSS) : null; }