Refine search
protected boolean equalTokenSequence(TokenSource first, TokenSource second) { Token token = null; while (!(token = first.nextToken()).equals(Token.EOF_TOKEN)) { Token otherToken = second.nextToken(); if (otherToken.equals(Token.EOF_TOKEN)) { return false; } if (!token.getText().equals(otherToken.getText())) { return false; } } return true; }
@Override public String getSourceName() { return "[IndentTokenSource] " + delegate.getSourceName(); //$NON-NLS-1$ }
Token t = tokenSource.nextToken(); while ( t!=null && t.getType()!=CharStream.EOF ) { boolean discard = false; Integer channelI = channelOverrideMap.get(t.getType()); if ( channelI!=null ) { t.setChannel(channelI); index++; t = tokenSource.nextToken();
/** add n elements to buffer */ protected void fetch(int n) { for (int i=1; i<=n; i++) { Token t = tokenSource.nextToken(); t.setTokenIndex(tokens.size()); //System.out.println("adding "+t+" at index "+tokens.size()); tokens.add(t); if ( t.getType()==Token.EOF ) break; } }
protected boolean isSameTokenSequence(TokenSource originalSource, TokenSource newSource, int expectedLength) { Token token = originalSource.nextToken(); int newLength = 0; while(Token.EOF != token.getType()) { Token newToken = newSource.nextToken(); if (token.getType() != newToken.getType()) { return false; } newLength += TokenTool.getLength(newToken); token = originalSource.nextToken(); } return newLength == expectedLength; }
@Override public Token nextElement() { Token t = tokenSource.nextToken(); t.setTokenIndex(tokenIndex++); return t; }
protected List<TokenInfo> createTokenInfos(String string) { List<TokenInfo> result = Lists.newArrayListWithExpectedSize(string.length() / 3); TokenSource source = createTokenSource(string); CommonToken token = (CommonToken) source.nextToken(); while (token != Token.EOF_TOKEN) { TokenInfo info = createTokenInfo(token); result.add(info); token = (CommonToken) source.nextToken(); } return result; }
Token t = tokenSource.nextToken(); while ( t!=null && t.getType()!=CharStream.EOF ) { boolean discard = false; Integer channelI = channelOverrideMap.get(t.getType()); if ( channelI!=null ) { t.setChannel(channelI); index++; t = tokenSource.nextToken();
/** add n elements to buffer */ protected void fetch(int n) { for (int i=1; i<=n; i++) { Token t = tokenSource.nextToken(); t.setTokenIndex(tokens.size()); //System.out.println("adding "+t+" at index "+tokens.size()); tokens.add(t); if ( t.getType()==Token.EOF ) break; } }
int currentTokenType; StringBuilder tokenString = new StringBuilder(); while((currentToken = stream.getTokenSource().nextToken()).getType() != DateLexer.EOF) { currentTokenType = currentToken.getType(); tokenString.append(DateParser.tokenNames[currentTokenType]).append(" "); builder.append("GROUP: "); for (Token token : group) { builder.append(DateParser.tokenNames[token.getType()]).append(" ");
@Override public Token nextElement() { Token t = tokenSource.nextToken(); t.setTokenIndex(tokenIndex++); return t; }
/** * Returns next token from the wrapped token source. Stores it in a list if * necessary. * */ @Override public CommonToken nextToken() { CommonToken nextToken = (CommonToken)source.nextToken(); if (shouldCollect(nextToken)) { collectedTokens.add(nextToken); } return nextToken; }
Token t = tokenSource.nextToken(); while ( t!=null && t.getType()!=CharStream.EOF ) { boolean discard = false; Integer channelI = channelOverrideMap.get(t.getType()); if ( channelI!=null ) { t.setChannel(channelI); index++; t = tokenSource.nextToken();
protected boolean equalTokenSequence(TokenSource first, TokenSource second) { Token token = null; while(!(token = first.nextToken()).equals(Token.EOF_TOKEN)) { Token otherToken = second.nextToken(); if (otherToken.equals(Token.EOF_TOKEN)) { return false; } if (!token.getText().equals(otherToken.getText())) { return false; } } return true; }
/** add n elements to buffer */ protected void fetch(int n) { for (int i=1; i<=n; i++) { Token t = tokenSource.nextToken(); t.setTokenIndex(tokens.size()); //System.out.println("adding "+t+" at index "+tokens.size()); tokens.add(t); if ( t.getType()==Token.EOF ) break; } }
int currentTokenType; StringBuilder tokenString = new StringBuilder(); while((currentToken = stream.getTokenSource().nextToken()).getType() != DateLexer.EOF) { currentTokenType = currentToken.getType(); tokenString.append(DateParser.tokenNames[currentTokenType]).append(" "); builder.append("GROUP: "); for (Token token : group) { builder.append(DateParser.tokenNames[token.getType()]).append(" ");
@Override public String getSourceName() { return source.getSourceName(); }
@Override public Token nextElement() { Token t = tokenSource.nextToken(); t.setTokenIndex(tokenIndex++); return t; }
public Token nextToken() { if (nextTokens.hasNext()) { Token result = nextTokens.next(); return result; } Token result = delegate.nextToken(); splitToken(result, nextTokens); result = nextTokens.next(); return result; }
Token t = tokenSource.nextToken(); while ( t!=null && t.getType()!=CharStream.EOF ) { boolean discard = false; Integer channelI = channelOverrideMap.get(t.getType()); if ( channelI!=null ) { t.setChannel(channelI); index++; t = tokenSource.nextToken();