/** * This method is used to get the first non-tight HTML tag encountered while parsing javadoc. * This shall eventually be reflected by the {@link ParseStatus} object returned by * {@link #parseJavadocAsDetailNode(DetailAST)} method via the instance member * {@link ParseStatus#firstNonTightHtmlTag}, and checks not supposed to process non-tight HTML * or the ones which are supposed to log violation for non-tight javadocs can utilize that. * * @param javadocParser The ANTLR recognizer instance which has been used to parse the javadoc * @return First non-tight HTML tag if one exists; null otherwise */ private Token getFirstNonTightHtmlTag(JavadocParser javadocParser) { final CommonToken offendingToken; final ParserRuleContext nonTightTagStartContext = javadocParser.nonTightTagStartContext; if (nonTightTagStartContext == null) { offendingToken = null; } else { final Token token = ((TerminalNode) nonTightTagStartContext.getChild(1)) .getSymbol(); offendingToken = new CommonToken(token); offendingToken.setLine(offendingToken.getLine() + errorListener.offset); } return offendingToken; }
@Override public Token nextToken() { if (!queue.isEmpty()) { return queue.poll(); } Token next = super.nextToken(); if (next.getType() != Word) { return next; } Token next2 = super.nextToken(); if (next2.getType() == Punctuation && next2.getText().equals(".")) { String abbrev = next.getText() + "."; if (abbreviations != null && abbreviations.contains(abbrev)) { CommonToken commonToken = new CommonToken(Abbreviation, abbrev); commonToken.setStartIndex(next.getStartIndex()); commonToken.setStopIndex(next2.getStopIndex()); commonToken.setTokenIndex(next.getTokenIndex()); commonToken.setCharPositionInLine(next.getCharPositionInLine()); commonToken.setLine(next.getLine()); return commonToken; } } queue.offer(next2); return next; }
@Override public CommonToken create(Pair<TokenSource, CharStream> source, int type, String text, int channel, int start, int stop, int line, int charPositionInLine) { CommonToken t = new CommonToken(source, type, channel, start, stop); t.setLine(line); t.setCharPositionInLine(charPositionInLine); if ( text!=null ) { t.setText(text); } else if ( copyText && source.b != null ) { t.setText(source.b.getText(Interval.of(start,stop))); } return t; }
t.setLine(line); return t;
private Token createDedent() { CommonToken dedent = commonToken(Python3Parser.DEDENT, ""); dedent.setLine(this.lastToken.getLine()); return dedent; }
private Token createDedent() { CommonToken dedent = commonToken(Python335Parser.DEDENT, ""); dedent.setLine(this.lastToken.getLine()); return dedent; }
private Token createDedent() { CommonToken dedent = commonToken(Python3Parser.DEDENT, ""); dedent.setLine(this.lastToken.getLine()); return dedent; }
private Token createDedent() { CommonToken dedent = commonToken(VyperParser.DEDENT, ""); dedent.setLine(this.lastToken.getLine()); return dedent; }
private Token createDedent() { CommonToken dedent = commonToken(ProgramParser.DEDENT, ""); dedent.setLine(this.lastToken.getLine()); return dedent; }
private Token createDedent() { CommonToken dedent = commonToken(Python3Parser.DEDENT, ""); dedent.setLine(this.lastToken.getLine()); return dedent; }
public ListTokenSource(List<Token> token) { this.token = token; Preconditions.checkNotNull(token); Preconditions.checkArgument(!token.isEmpty(), "Internal token list must not be empty"); for(Token t : token) { if(t.getTokenSource() != null) { this.factory = t.getTokenSource().getTokenFactory(); break; } } Preconditions.checkNotNull(this.factory, "Internal token list needs a valid TokenSource"); Token lastToken = token.get(token.size()-1); eofToken.setLine(lastToken.getLine()); eofToken.setCharPositionInLine(lastToken.getCharPositionInLine()); }
@Override public void syntaxError(Recognizer<?, ?> recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) { if (!REPORT_SYNTAX_ERRORS) { return; } String sourceName = recognizer.getInputStream().getSourceName(); if (!sourceName.isEmpty()) { sourceName = String.format("%s:%d:%d: ", sourceName, line, charPositionInLine); } final CommonToken errorToken = new CommonToken(0); errorToken.setLine(line); errorToken.setCharPositionInLine(charPositionInLine); ErrorLogger.error(ErrorIncidenceType.Parse, errorToken, "column ", Integer.toString(charPositionInLine), ": ", msg); } }
private static void reinitializeObject(final Type objectType, final StaticScope objectsScope) { final CommonToken objectToken = new CommonToken(0); objectToken.setLine(157239); final ClassDeclaration objectClass = new ClassDeclaration("Object", objectsScope, null, objectToken); globalScope_.declareClass(objectClass); objectClass.setType(objectType); objectsScope.setDeclaration(objectClass); typeDeclarationList_.add(objectClass); }
/** * This method is used to get the first non-tight HTML tag encountered while parsing javadoc. * This shall eventually be reflected by the {@link ParseStatus} object returned by * {@link #parseJavadocAsDetailNode(DetailAST)} method via the instance member * {@link ParseStatus#firstNonTightHtmlTag}, and checks not supposed to process non-tight HTML * or the ones which are supposed to log violation for non-tight javadocs can utilize that. * * @param javadocParser The ANTLR recognizer instance which has been used to parse the javadoc * @return First non-tight HTML tag if one exists; null otherwise */ private Token getFirstNonTightHtmlTag(JavadocParser javadocParser) { final CommonToken offendingToken; final ParserRuleContext nonTightTagStartContext = javadocParser.nonTightTagStartContext; if (nonTightTagStartContext == null) { offendingToken = null; } else { final Token token = ((TerminalNode) nonTightTagStartContext.getChild(1)) .getSymbol(); offendingToken = new CommonToken(token); offendingToken.setLine(offendingToken.getLine() + errorListener.offset); } return offendingToken; }
public static void wipeCharPositionInfoAndWhitespaceTokens(CodeBuffTokenStream tokens) { tokens.fill(); CommonToken dummy = new CommonToken(Token.INVALID_TYPE, ""); dummy.setChannel(Token.HIDDEN_CHANNEL); Token firstRealToken = tokens.getNextRealToken(-1); for (int i = 0; i<tokens.size(); i++) { if ( i==firstRealToken.getTokenIndex() ) continue; // don't wack first token CommonToken t = (CommonToken)tokens.get(i); if ( t.getText().matches("\\s+") ) { tokens.getTokens().set(i, dummy); // wack whitespace token so we can't use it during prediction } else { t.setLine(0); t.setCharPositionInLine(-1); } } }
@Override public CommonToken create(Pair<TokenSource, CharStream> source, int type, String text, int channel, int start, int stop, int line, int charPositionInLine) { CommonToken t = new CommonToken(source, type, channel, start, stop); t.setLine(line); t.setCharPositionInLine(charPositionInLine); if ( text!=null ) { t.setText(text); } else if ( copyText && source.b != null ) { t.setText(source.b.getText(Interval.of(start,stop))); } return t; }
@Override public CommonToken create(Pair<TokenSource, CharStream> source, int type, String text, int channel, int start, int stop, int line, int charPositionInLine) { CommonToken t = new CommonToken(source, type, channel, start, stop); t.setLine(line); t.setCharPositionInLine(charPositionInLine); if ( text!=null ) { t.setText(text); } else if ( copyText && source.b != null ) { t.setText(source.b.getText(Interval.of(start,stop))); } return t; }
@Override public CommonToken create(Pair<TokenSource, CharStream> source, int type, String text, int channel, int start, int stop, int line, int charPositionInLine) { CommonToken t = new CommonToken(source, type, channel, start, stop); t.setLine(line); t.setCharPositionInLine(charPositionInLine); if ( text!=null ) { t.setText(text); } else if ( copyText && source.b != null ) { t.setText(source.b.getText(Interval.of(start,stop))); } return t; }
@Override public CommonToken create(Tuple2<? extends TokenSource, CharStream> source, int type, String text, int channel, int start, int stop, int line, int charPositionInLine) { CommonToken t = new CommonToken(source, type, channel, start, stop); t.setLine(line); t.setCharPositionInLine(charPositionInLine); if ( text!=null ) { t.setText(text); } else if ( copyText && source.getItem2() != null ) { t.setText(source.getItem2().getText(Interval.of(start,stop))); } return t; }
} else { final CommonToken errorToken = new CommonToken(0); errorToken.setLine(1); errorHook5p2(ErrorIncidenceType.Fatal, errorToken, "Missing main expression.", " Did you enter any program at all?", "", "");