private String applyReplacements(List<Replacement> replacements, TemplateOptions options) throws IOException { StringWriter sw = new StringWriter(); reconstruct(sw, tokenStream, 0, tokenStream.size() - 1, replacements, options); return sw.toString(); }
private String applyReplacements(List<Replacement> replacements, TemplateOptions options) throws IOException { StringWriter sw = new StringWriter(); reconstruct(sw, tokenStream, 0, tokenStream.size() - 1, replacements, options); return sw.toString(); }
/** * Print the token stream to the logger. * * @param tokens to print */ public static void printTokens(CommonTokenStream tokens) { if (log.isDebugEnabled()) { List tokenList = tokens.getTokens(); StringWriter writer = new StringWriter(); PrintWriter printer = new PrintWriter(writer); for (int i = 0; i < tokens.size(); i++) { Token t = (Token) tokenList.get(i); String text = t.getText(); if (text.trim().length() == 0) { printer.print("'" + text + "'"); } else { printer.print(text); } printer.print('['); printer.print(t.getType()); printer.print(']'); printer.print(" "); } printer.println(); log.debug("Tokens: " + writer.toString()); } }
/** * Print the token stream to the logger. * * @param tokens to print */ public static void printTokens(CommonTokenStream tokens) { if (log.isDebugEnabled()) { List tokenList = tokens.getTokens(); StringWriter writer = new StringWriter(); PrintWriter printer = new PrintWriter(writer); for (int i = 0; i < tokens.size(); i++) { Token t = (Token) tokenList.get(i); String text = t.getText(); if (text.trim().length() == 0) { printer.print("'" + text + "'"); } else { printer.print(text); } printer.print('['); printer.print(t.getType()); printer.print(']'); printer.print(" "); } printer.println(); log.debug("Tokens: " + writer.toString()); } }
private static int findStartTokenScript(int startIndex, CommonTokenStream tokens, int tokenTypeSearch) { int found = -1; for (int i = startIndex; i < tokens.size(); i++) { if (tokens.get(i).getType() == tokenTypeSearch) { return i; } } return found; }
public static List<Token> getRealTokens(CommonTokenStream tokens) { List<Token> real = new ArrayList<>(); for (int i=0; i<tokens.size(); i++) { Token t = tokens.get(i); if ( t.getType()!=Token.EOF && t.getChannel()==Lexer.DEFAULT_TOKEN_CHANNEL ) { real.add(t); } } return real; }
private static int findStartTokenScript(int startIndex, CommonTokenStream tokens, int tokenTypeSearch) { int found = -1; for (int i = startIndex; i < tokens.size(); i++) { if (tokens.get(i).getType() == tokenTypeSearch) { return i; } } return found; }
public static Token previousRealToken(CommonTokenStream tokens, int i) { int size = tokens.size(); i--; // search before current i token if ( i>=size || i<0 ) return null; Token t = tokens.get(i); while ( t.getChannel()==Token.HIDDEN_CHANNEL ) { i--; if ( i<0 ) return null; t = tokens.get(i); } return t; }
for (int i = startIndex; i < tokens.size(); i++) { if (tokens.get(i).getType() == tokenTypeSearch) { for (int j = i + 1; j < tokens.size(); j++) { Token next = tokens.get(j); if (next.getChannel() == 0) { for (int i = startIndex; i < tokens.size(); i++) { if (tokens.get(i).getType() == tokenTypeSearch) { indexLast = i;
private static boolean isContainsScriptExpression(CommonTokenStream tokens) { for (int i = 0; i < tokens.size(); i++) { if (tokens.get(i).getType() == EsperEPL2GrammarParser.EXPRESSIONDECL) { int startTokenLcurly = findStartTokenScript(i + 1, tokens, EsperEPL2GrammarParser.LCURLY); int startTokenLbrack = findStartTokenScript(i + 1, tokens, EsperEPL2GrammarParser.LBRACK); // Handle: // expression ABC { some[other] } // expression boolean js:doit(...) [ {} ] if (startTokenLbrack != -1 && (startTokenLcurly == -1 || startTokenLcurly > startTokenLbrack)) { return true; } } } return false; }
private static boolean isContainsScriptExpression(CommonTokenStream tokens) { for (int i = 0; i < tokens.size(); i++) { if (tokens.get(i).getType() == EsperEPL2GrammarParser.EXPRESSIONDECL) { int startTokenLcurly = findStartTokenScript(i + 1, tokens, EsperEPL2GrammarParser.LCURLY); int startTokenLbrack = findStartTokenScript(i + 1, tokens, EsperEPL2GrammarParser.LBRACK); // Handle: // expression ABC { some[other] } // expression boolean js:doit(...) [ {} ] if (startTokenLbrack != -1 && (startTokenLcurly == -1 || startTokenLcurly > startTokenLbrack)) { return true; } } } return false; }
private static ScriptResult rewriteTokensScript(CommonTokenStream tokens) { List<String> scripts = new ArrayList<String>(); List<UniformPair<Integer>> scriptTokenIndexRanges = new ArrayList<UniformPair<Integer>>(); for (int i = 0; i < tokens.size(); i++) { if (tokens.get(i).getType() == EsperEPL2GrammarParser.EXPRESSIONDECL) { Token tokenBefore = getTokenBefore(i, tokens); boolean isCreateExpressionClause = tokenBefore != null && tokenBefore.getType() == EsperEPL2GrammarParser.CREATE; Pair<String, Integer> nameAndNameStart = findScriptName(i + 1, tokens); int startIndex = findStartTokenScript(nameAndNameStart.getSecond(), tokens, EsperEPL2GrammarParser.LBRACK); if (startIndex != -1) { int endIndex = findEndTokenScript(startIndex + 1, tokens, EsperEPL2GrammarParser.RBRACK, EsperEPL2GrammarParser.getAfterScriptTokens(), !isCreateExpressionClause); if (endIndex != -1) { StringWriter writer = new StringWriter(); for (int j = startIndex + 1; j < endIndex; j++) { writer.append(tokens.get(j).getText()); } scripts.add(writer.toString()); scriptTokenIndexRanges.add(new UniformPair<Integer>(startIndex, endIndex)); } } } } String rewrittenEPL = rewriteScripts(scriptTokenIndexRanges, tokens); return new ScriptResult(rewrittenEPL, scripts); }
private static ScriptResult rewriteTokensScript(CommonTokenStream tokens) { List<String> scripts = new ArrayList<String>(); List<UniformPair<Integer>> scriptTokenIndexRanges = new ArrayList<UniformPair<Integer>>(); for (int i = 0; i < tokens.size(); i++) { if (tokens.get(i).getType() == EsperEPL2GrammarParser.EXPRESSIONDECL) { Token tokenBefore = getTokenBefore(i, tokens); boolean isCreateExpressionClause = tokenBefore != null && tokenBefore.getType() == EsperEPL2GrammarParser.CREATE; Pair<String, Integer> nameAndNameStart = findScriptName(i + 1, tokens); int startIndex = findStartTokenScript(nameAndNameStart.getSecond(), tokens, EsperEPL2GrammarParser.LBRACK); if (startIndex != -1) { int endIndex = findEndTokenScript(startIndex + 1, tokens, EsperEPL2GrammarParser.RBRACK, EsperEPL2GrammarParser.getAfterScriptTokens(), !isCreateExpressionClause); if (endIndex != -1) { StringWriter writer = new StringWriter(); for (int j = startIndex + 1; j < endIndex; j++) { writer.append(tokens.get(j).getText()); } scripts.add(writer.toString()); scriptTokenIndexRanges.add(new UniformPair<Integer>(startIndex, endIndex)); } } } } String rewrittenEPL = rewriteScripts(scriptTokenIndexRanges, tokens); return new ScriptResult(rewrittenEPL, scripts); }
@Override public Tree parse(String slangCode) { SLangLexer lexer = new SLangLexer(CharStreams.fromString(slangCode)); List<Comment> comments = new ArrayList<>(); CommonTokenStream antlrTokens = new CommonTokenStream(lexer); antlrTokens.fill(); List<org.sonarsource.slang.api.Token> tokens = new ArrayList<>(); for (int index = 0; index < antlrTokens.size(); index++) { Token token = antlrTokens.get(index); TextRange textRange = getSlangTextRange(token); if (token.getChannel() == 1) { comments.add(comment(token, textRange)); } else { Type type = Type.OTHER; if (KEYWORD_TOKEN_TYPES.contains(token.getType())) { type = Type.KEYWORD; } else if (token.getType() == SLangParser.StringLiteral) { type = Type.STRING_LITERAL; } tokens.add(new TokenImpl(textRange, token.getText(), type)); } } SLangParser parser = new SLangParser(antlrTokens); parser.setErrorHandler(new ErrorStrategy()); SLangParseTreeVisitor slangVisitor = new SLangParseTreeVisitor(comments, tokens); return slangVisitor.visit(parser.slangFile()); }
private static Pair<String, Integer> findScriptName(int start, CommonTokenStream tokens) { String lastIdent = null; int lastIdentIndex = 0; for (int i = start; i < tokens.size(); i++) { if (tokens.get(i).getType() == EsperEPL2GrammarParser.IDENT) { lastIdent = tokens.get(i).getText(); lastIdentIndex = i; } if (tokens.get(i).getType() == EsperEPL2GrammarParser.LPAREN) { break; } // find beginning of script, ignore brackets if (tokens.get(i).getType() == EsperEPL2GrammarParser.LBRACK && tokens.get(i + 1).getType() != EsperEPL2GrammarParser.RBRACK) { break; } } if (lastIdent == null) { throw new IllegalStateException("Failed to parse expression name"); } return new Pair<String, Integer>(lastIdent, lastIdentIndex); }
private static Pair<String, Integer> findScriptName(int start, CommonTokenStream tokens) { String lastIdent = null; int lastIdentIndex = 0; for (int i = start; i < tokens.size(); i++) { if (tokens.get(i).getType() == EsperEPL2GrammarParser.IDENT) { lastIdent = tokens.get(i).getText(); lastIdentIndex = i; } if (tokens.get(i).getType() == EsperEPL2GrammarParser.LPAREN) { break; } // find beginning of script, ignore brackets if (tokens.get(i).getType() == EsperEPL2GrammarParser.LBRACK && tokens.get(i + 1).getType() != EsperEPL2GrammarParser.RBRACK) { break; } } if (lastIdent == null) { throw new IllegalStateException("Failed to parse expression name"); } return new Pair<String, Integer>(lastIdent, lastIdentIndex); }
public static Token nextRealToken(CommonTokenStream tokens, int i) { int n = tokens.size(); i++; // search after current i token if ( i>=n || i<0 ) return null; Token t = tokens.get(i); while ( t.getChannel()==Token.HIDDEN_CHANNEL ) { if ( t.getType()==Token.EOF ) { TokenSource tokenSource = tokens.getTokenSource(); if ( tokenSource==null ) { return new CommonToken(Token.EOF, "EOF"); } TokenFactory<?> tokenFactory = tokenSource.getTokenFactory(); if ( tokenFactory==null ) { return new CommonToken(Token.EOF, "EOF"); } return tokenFactory.create(Token.EOF, "EOF"); } i++; if ( i>=n ) return null; // just in case no EOF t = tokens.get(i); } return t; }
@Test public void test(){ CommonTokenStream ts = TokenStreamFactory.createTokenStream("class{ }"); int tokenSize = ts.size(); assertEquals(0, tokenSize); List<Token> tokens = ts.getTokens(); assertEquals(0, tokens.size()); ts.consume(); ts.consume(); assertEquals("}", ts.LT(1).getText()); assertEquals("{", ts.LT(-1).getText()); assertEquals("class", ts.LT(-2).getText()); //why is it 4? assertEquals(4, ts.size()); int consumeSize = 2; while(ts.LA(1)!=IntStream.EOF){ ts.consume(); consumeSize++; } tokens = ts.getTokens(); assertEquals(5, tokens.size()); assertEquals(3, consumeSize); }
@Test public void test(){ KalangCompiler kc = new KalangCompiler(){ @Override public CodeGenerator createCodeGenerator(CompilationUnit compilationUnit) { return new Ast2JavaStub(); } }; kc.addSource("Test", "class{ }","Test.kl"); kc.compile(); CompilationUnit unit = kc.getCompilationUnit("Test"); assert unit != null; CommonTokenStream ts = unit.getTokenStream(); //the tokens contains tokens in all channels List<Token> tokens = ts.getTokens(); assertEquals(5, ts.size()); testTokenNavigator(tokens.toArray(new Token[0]),unit.getAstBuilder().getParseTree()); }