protected SourceLocation getSourceLocation(Token token) { String sourceName = token.getTokenSource().getSourceName(); if (IntStream.UNKNOWN_SOURCE_NAME.equals(sourceName)) { // UNKNOWN_SOURCE_NAME is Antrl's way of indicating that no source name was given during parsing -- // which is the case when queries and other operations are parsed. We don't want this hardcoded // '<unknown>' sourceName to leak to clients when the response is serialized as JSON, so we null it. sourceName = null; } return new SourceLocation(token.getLine(), token.getCharPositionInLine() + 1, sourceName); }
@Override public void exitNonReserved(SqlBaseParser.NonReservedContext context) { // we can't modify the tree during rule enter/exit event handling unless we're dealing with a terminal. // Otherwise, ANTLR gets confused an fires spurious notifications. if (!(context.getChild(0) instanceof TerminalNode)) { int rule = ((ParserRuleContext) context.getChild(0)).getRuleIndex(); throw new AssertionError("nonReserved can only contain tokens. Found nested rule: " + ruleNames.get(rule)); } // replace nonReserved words with IDENT tokens context.getParent().removeLastChild(); Token token = (Token) context.getChild(0).getPayload(); context.getParent().addChild(new CommonToken( new Pair<>(token.getTokenSource(), token.getInputStream()), SqlBaseLexer.IDENTIFIER, token.getChannel(), token.getStartIndex(), token.getStopIndex())); } }
getTokenFactory().create(new Pair<TokenSource, CharStream>(tok.getTokenSource(), tok.getTokenSource().getInputStream()), expectedTokenType, tok.getText(), Token.DEFAULT_CHANNEL, Token tok = e.getOffendingToken(); Token errToken = getTokenFactory().create(new Pair<TokenSource, CharStream>(tok.getTokenSource(), tok.getTokenSource().getInputStream()), Token.INVALID_TYPE, tok.getText(), Token.DEFAULT_CHANNEL,
/** * Constructs a new {@link CommonToken} as a copy of another {@link Token}. * * <p> * If {@code oldToken} is also a {@link CommonToken} instance, the newly * constructed token will share a reference to the {@link #text} field and * the {@link Pair} stored in {@link #source}. Otherwise, {@link #text} will * be assigned the result of calling {@link #getText}, and {@link #source} * will be constructed from the result of {@link Token#getTokenSource} and * {@link Token#getInputStream}.</p> * * @param oldToken The token to copy. */ public CommonToken(Token oldToken) { type = oldToken.getType(); line = oldToken.getLine(); index = oldToken.getTokenIndex(); charPositionInLine = oldToken.getCharPositionInLine(); channel = oldToken.getChannel(); start = oldToken.getStartIndex(); stop = oldToken.getStopIndex(); if (oldToken instanceof CommonToken) { text = ((CommonToken)oldToken).text; source = ((CommonToken)oldToken).source; } else { text = oldToken.getText(); source = new Pair<TokenSource, CharStream>(oldToken.getTokenSource(), oldToken.getInputStream()); } }
recognizer.getTokenFactory().create(new Pair<TokenSource, CharStream>(current.getTokenSource(), current.getTokenSource().getInputStream()), expectedTokenType, tokenText, Token.DEFAULT_CHANNEL, -1, -1,
@Override public void enterUserAgent(UserAgentContext ctx) { // In case of a parse error the 'parsed' version of agent can be incomplete inform(ctx, AGENT, ctx.start.getTokenSource().getInputStream().toString()); }
public ListTokenSource(List<Token> token) { this.token = token; Preconditions.checkNotNull(token); Preconditions.checkArgument(!token.isEmpty(), "Internal token list must not be empty"); for(Token t : token) { if(t.getTokenSource() != null) { this.factory = t.getTokenSource().getTokenFactory(); break; } } Preconditions.checkNotNull(this.factory, "Internal token list needs a valid TokenSource"); Token lastToken = token.get(token.size()-1); eofToken.setLine(lastToken.getLine()); eofToken.setCharPositionInLine(lastToken.getCharPositionInLine()); }
@Override public void reportUnwantedToken(Parser parser) { if (!parser.getContext().start.getTokenSource().getSourceName() .equals(context.get(DocumentServiceKeys.RELATIVE_FILE_PATH_KEY).replace("\\", "/"))) { return; } this.context.put(CompletionKeys.TOKEN_STREAM_KEY, parser.getTokenStream()); }
@Override public void reportInputMismatch(Parser parser, InputMismatchException e) { if (!parser.getContext().start.getTokenSource().getSourceName() .equals(context.get(DocumentServiceKeys.RELATIVE_FILE_PATH_KEY).replace("\\", "/"))) { return; } this.context.put(CompletionKeys.TOKEN_STREAM_KEY, parser.getTokenStream()); }
@Override public void reportMissingToken(Parser parser) { if (!parser.getContext().start.getTokenSource().getSourceName() .equals(context.get(DocumentServiceKeys.RELATIVE_FILE_PATH_KEY).replace("\\", "/"))) { return; } this.context.put(CompletionKeys.TOKEN_STREAM_KEY, parser.getTokenStream()); }
@Override public void reportNoViableAlternative(Parser parser, NoViableAltException e) { if (!parser.getContext().start.getTokenSource().getSourceName() .equals(context.get(DocumentServiceKeys.RELATIVE_FILE_PATH_KEY).replace("\\", "/"))) { return; } this.context.put(CompletionKeys.TOKEN_STREAM_KEY, parser.getTokenStream()); }
protected Token constructToken(TokenSource tokenSource, int expectedTokenType, String tokenText, Token current) { TokenFactory factory = tokenSource.getTokenFactory(); return factory.create(Tuple.create(tokenSource, current.getTokenSource().getInputStream()), expectedTokenType, tokenText, Token.DEFAULT_CHANNEL, -1, -1, current.getLine(), current.getCharPositionInLine()); }
@Override public void reportMatch(Parser recognizer) { if (!recognizer.getContext().start.getTokenSource().getSourceName() .equals(context.get(DocumentServiceKeys.RELATIVE_FILE_PATH_KEY).replace("\\", "/"))) { super.reportMatch(recognizer); return; } removePendingTokensAfterThisToken(recognizer, removeStartToken, TokenRemovalStrategy.MATCH); boolean inFirstTokenOfCursorLine = isInFirstTokenOfCursorLine(recognizer); boolean inLastTermination = isInLastTermination(recognizer); super.reportMatch(recognizer); if (recognizer.getCurrentToken().getType() != BallerinaParser.EOF && inLastTermination) { deleteTokensUpToCursor(recognizer, inLastTermination, inFirstTokenOfCursorLine); } }
@Override public void sync(Parser recognizer) throws RecognitionException { if (!recognizer.getContext().start.getTokenSource().getSourceName() .equals(context.get(DocumentServiceKeys.RELATIVE_FILE_PATH_KEY).replace("\\", "/"))) { super.sync(recognizer); return; } removePendingTokensAfterThisToken(recognizer, removeStartToken, TokenRemovalStrategy.SYNC); boolean inFirstTokenOfCursorLine = isInFirstTokenOfCursorLine(recognizer); boolean inLastTermination = isInLastTermination(recognizer); if (recognizer.getCurrentToken().getType() != BallerinaParser.EOF && inFirstTokenOfCursorLine) { deleteTokensUpToCursor(recognizer, inLastTermination, inFirstTokenOfCursorLine); } else if (recognizer.getCurrentToken().getType() != BallerinaParser.EOF && inLastTermination) { deleteTokensUpToCursor(recognizer, inLastTermination, inFirstTokenOfCursorLine); } super.sync(recognizer); }
@Override public void exitNonReserved(SqlBaseParser.NonReservedContext context) { // we can't modify the tree during rule enter/exit event handling unless we're dealing with a terminal. // Otherwise, ANTLR gets confused an fires spurious notifications. if (!(context.getChild(0) instanceof TerminalNode)) { int rule = ((ParserRuleContext) context.getChild(0)).getRuleIndex(); throw new AssertionError( "nonReserved can only contain tokens. Found nested rule: " + ruleNames .get(rule)); } // replace nonReserved words with IDENT tokens context.getParent().removeLastChild(); Token token = (Token) context.getChild(0).getPayload(); context.getParent().addChild(new CommonToken( new Pair<>(token.getTokenSource(), token.getInputStream()), SqlBaseLexer.IDENTIFIER, token.getChannel(), token.getStartIndex(), token.getStopIndex())); } }
@Override public void exitNonReserved(SqlBaseParser.NonReservedContext context) { // we can't modify the tree during rule enter/exit event handling unless we're dealing with a terminal. // Otherwise, ANTLR gets confused an fires spurious notifications. if (!(context.getChild(0) instanceof TerminalNode)) { int rule = ((ParserRuleContext) context.getChild(0)).getRuleIndex(); throw new AssertionError("nonReserved can only contain tokens. Found nested rule: " + ruleNames.get(rule)); } // replace nonReserved words with IDENT tokens context.getParent().removeLastChild(); Token token = (Token) context.getChild(0).getPayload(); context.getParent().addChild(new CommonToken( new Pair<>(token.getTokenSource(), token.getInputStream()), SqlBaseLexer.IDENTIFIER, token.getChannel(), token.getStartIndex(), token.getStopIndex())); } }
@Override public void exitNonReserved(SqlBaseParser.NonReservedContext context) { // we can't modify the tree during rule enter/exit event handling unless we're dealing with a terminal. // Otherwise, ANTLR gets confused an fires spurious notifications. if (!(context.getChild(0) instanceof TerminalNode)) { int rule = ((ParserRuleContext) context.getChild(0)).getRuleIndex(); throw new AssertionError("nonReserved can only contain tokens. Found nested rule: " + ruleNames.get(rule)); } // replace nonReserved words with IDENT tokens context.getParent().removeLastChild(); Token token = (Token) context.getChild(0).getPayload(); context.getParent().addChild(new CommonToken( new Pair<>(token.getTokenSource(), token.getInputStream()), SqlBaseLexer.IDENTIFIER, token.getChannel(), token.getStartIndex(), token.getStopIndex())); } }
@Override public void exitNonReserved(BQLBaseParser.NonReservedContext context) { // We can't modify the tree during rule enter/exit event handling unless we're dealing with a terminal. // Otherwise, ANTLR gets confused an fires spurious notifications. if (!(context.getChild(0) instanceof TerminalNode)) { int rule = ((ParserRuleContext) context.getChild(0)).getRuleIndex(); throw new AssertionError("NonReserved can only contain tokens. Found nested rule: " + ruleNames.get(rule)); } // Replace nonReserved words with IDENT tokens. context.getParent().removeLastChild(); Token token = (Token) context.getChild(0).getPayload(); context.getParent().addChild(new CommonToken( new Pair<>(token.getTokenSource(), token.getInputStream()), BQLBaseLexer.IDENTIFIER, token.getChannel(), token.getStartIndex(), token.getStopIndex())); } }
@Override public void exitQuotedIdentifier(SqlBaseParser.QuotedIdentifierContext context) { // Remove quotes context.getParent().removeLastChild(); Token token = (Token) context.getChild(0).getPayload(); context.getParent().addChild(new CommonToken( new Pair<>(token.getTokenSource(), token.getInputStream()), SqlBaseLexer.IDENTIFIER, token.getChannel(), token.getStartIndex() + 1, token.getStopIndex() - 1)); }
@Override public void exitNonReserved(SqlBaseParser.NonReservedContext context) { // replace nonReserved words with IDENT tokens context.getParent().removeLastChild(); Token token = (Token) context.getChild(0).getPayload(); context.getParent().addChild(new CommonToken( new Pair<>(token.getTokenSource(), token.getInputStream()), SqlBaseLexer.IDENTIFIER, token.getChannel(), token.getStartIndex(), token.getStopIndex())); } }