/** Switch to specified lex state. */ public void SwitchTo(int lexState) { if (lexState >= 19 || lexState < 0) throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", TokenMgrError.INVALID_LEXICAL_STATE); else curLexState = lexState; }
/** Switch to specified lex state. */ public void SwitchTo(int lexState) { if (lexState >= 4 || lexState < 0) throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", TokenMgrError.INVALID_LEXICAL_STATE); else curLexState = lexState; }
/** Switch to specified lex state. */ public void SwitchTo(int lexState) { if (lexState >= 21 || lexState < 0) throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", TokenMgrError.INVALID_LEXICAL_STATE); else curLexState = lexState; }
/** Switch to specified lex state. */ public void SwitchTo(int lexState) { if (lexState >= 4 || lexState < 0) throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", TokenMgrError.INVALID_LEXICAL_STATE); else curLexState = lexState; }
/** Switch to specified lex state. */ public void SwitchTo(int lexState) { if (lexState >= 6 || lexState < 0) throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", TokenMgrError.INVALID_LEXICAL_STATE); else curLexState = lexState; }
/** Switch to specified lex state. */ public void SwitchTo(int lexState) { if (lexState >= 19 || lexState < 0) throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", TokenMgrError.INVALID_LEXICAL_STATE); else curLexState = lexState; }
/** Switch to specified lex state. */ public void SwitchTo(int lexState) { if (lexState >= 5 || lexState < 0) throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", TokenMgrError.INVALID_LEXICAL_STATE); else curLexState = lexState; }
/** Switch to specified lex state. */ public void SwitchTo(int lexState) { if (lexState >= 3 || lexState < 0) throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", TokenMgrError.INVALID_LEXICAL_STATE); else curLexState = lexState; }
@Override public void emitErrorMessage(String msg) { throw new TokenMgrError(msg, TokenMgrError.LEXICAL_ERROR); } };
/** Switch to specified lex state. */ public void SwitchTo(int lexState) { if (lexState >= 3 || lexState < 0) throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", TokenMgrError.INVALID_LEXICAL_STATE); else curLexState = lexState; }
@Override public void tokenize(SourceCode source, Tokens cpdTokens) { String filename = source.getFileName(); try { Lexer lexer = new Lexer(); List<Token> tokens = lexer.getTokensOfFile(filename); for (Token token : tokens) { String tokenVal = token.tokenVal() != null ? token.tokenVal() : Integer.toString(token.tokenType()); TokenEntry cpdToken = new TokenEntry(tokenVal, filename, token.line()); cpdTokens.add(cpdToken); } cpdTokens.add(TokenEntry.getEOF()); } catch (RuntimeException e) { e.printStackTrace(); // Wrap exceptions of the Scala tokenizer in a TokenMgrError, so // they are correctly handled // when CPD is executed with the '--skipLexicalErrors' command line // option throw new TokenMgrError( "Lexical error in file " + filename + ". The scala tokenizer exited with error: " + e.getMessage(), TokenMgrError.LEXICAL_ERROR); } }
@Override public void tokenize(SourceCode sourceCode, Tokens tokenEntries) { StringBuilder buffer = sourceCode.getCodeBuffer(); GroovyLexer lexer = new GroovyLexer(new StringReader(buffer.toString())); TokenStream tokenStream = lexer.plumb(); try { Token token = tokenStream.nextToken(); while (token.getType() != Token.EOF_TYPE) { TokenEntry tokenEntry = new TokenEntry(token.getText(), sourceCode.getFileName(), token.getLine()); tokenEntries.add(tokenEntry); token = tokenStream.nextToken(); } } catch (TokenStreamException err) { // Wrap exceptions of the Groovy tokenizer in a TokenMgrError, so // they are correctly handled // when CPD is executed with the '--skipLexicalErrors' command line // option throw new TokenMgrError("Lexical error in file " + sourceCode.getFileName() + " at line " + lexer.getLine() + ", column " + lexer.getColumn() + ". Encountered: " + err.getMessage(), TokenMgrError.LEXICAL_ERROR); } finally { tokenEntries.add(TokenEntry.getEOF()); } } }
@Override public void tokenize(final SourceCode sourceCode, final Tokens tokenEntries) { AntlrTokenManager tokenManager = getLexerForSource(sourceCode); try { AntlrToken token = (AntlrToken) tokenManager.getNextToken(); while (token.getType() != Token.EOF) { if (!token.isHidden()) { final TokenEntry tokenEntry = new TokenEntry(token.getImage(), tokenManager.getFileName(), token.getBeginLine()); tokenEntries.add(tokenEntry); } token = (AntlrToken) tokenManager.getNextToken(); } } catch (final AntlrTokenManager.ANTLRSyntaxError err) { // Wrap exceptions of the ANTLR tokenizer in a TokenMgrError, so they are correctly handled // when CPD is executed with the '--skipLexicalErrors' command line option throw new TokenMgrError("Lexical error in file " + tokenManager.getFileName() + " at line " + err.getLine() + ", column " + err.getColumn() + ". Encountered: " + err.getMessage(), TokenMgrError.LEXICAL_ERROR); } finally { tokenEntries.add(TokenEntry.getEOF()); } }
error_after = curPos <= 1 ? "" : input_stream.GetImage(); throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR);
error_after = curPos <= 1 ? "" : input_stream.GetImage(); throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR);
error_after = curPos <= 1 ? "" : input_stream.GetImage(); throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR);
error_after = curPos <= 1 ? "" : input_stream.GetImage(); throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR);
error_after = curPos <= 1 ? "" : input_stream.GetImage(); throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR);
error_after = curPos <= 1 ? "" : input_stream.GetImage(); throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR);
error_after = curPos <= 1 ? "" : input_stream.GetImage(); throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR);