Refine search
public void expect(String s, int... tokens) { Lexer lexer = createLexer(s); for (int token : tokens) { Token t = lexer.nextToken(); assertThat(t.getType()).isEqualTo(token) .withFailMessage("Expected %s, got %s, with '%s'", nameOf(token), nameOf(t.getType()), t.getText()); } }
public boolean alreadyParsedRule(IntStream input, int ruleIndex) { if ( state.backtracking>1 ) return super.alreadyParsedRule(input, ruleIndex); return false; } // $ANTLR start "QUOTE"
public void memoize(IntStream input, int ruleIndex, int ruleStartIndex) { if ( state.backtracking>1 ) super.memoize(input, ruleIndex, ruleStartIndex); }
public void match(int c) throws MismatchedTokenException { if ( input.LA(1)!=c ) { if ( state.backtracking>0 ) { state.failed = true; return; } MismatchedTokenException mte = new MismatchedTokenException(c, input); recover(mte); // don't really recover; just consume in lexer throw mte; } input.consume(); state.failed = false; }
state.token = null; state.channel = Token.DEFAULT_CHANNEL; state.tokenStartCharIndex = input.index(); state.tokenStartCharPositionInLine = input.getCharPositionInLine(); state.tokenStartLine = input.getLine(); state.text = null; if ( input.LA(1)==CharStream.EOF ) { return getEOFToken(); mTokens(); if ( state.token==null ) { emit(); reportError(re); reportError(re); reportError(re); recover(re); // throw out current char and try again
state.token = null; state.channel = Token.DEFAULT_CHANNEL; state.tokenStartCharIndex = input.index(); state.tokenStartCharPositionInLine = input.getCharPositionInLine(); state.tokenStartLine = input.getLine(); state.text = null; if ( input.LA(1)==CharStream.EOF ) { Token.DEFAULT_CHANNEL, input.index(),input.index()); eof.setLine(getLine()); eof.setCharPositionInLine(getCharPositionInLine()); return eof; mTokens(); if ( state.token==null ) { emit(); reportError(nva); recover(nva); // throw out current char and try again reportError(re);
public Token nextToken(){ //The following code was pulled out from super.nextToken() if (input.index() == 0) { try { state.token = null; state.channel = Token.DEFAULT_CHANNEL; state.tokenStartCharIndex = input.index(); state.tokenStartCharPositionInLine = input.getCharPositionInLine(); state.tokenStartLine = input.getLine(); state.text = null; mFirstBodyString(); state.type = BodyString; emit(); return state.token; } catch (NoViableAltException nva) { reportError(nva); recover(nva); // throw out current char and try again } catch (RecognitionException re) { reportError(re); // match() routine has already called recover() } } return super.nextToken(); }
public Token nextToken(){ Token token = super.nextToken(); if(token.getType() == Token.EOF && !includes.empty()){ // We've got EOF and have non empty stack. SaveStruct ss = includes.pop(); setCharStream(ss.input); input.rewind(ss.marker); //this should be used instead of super [like below] to handle exits from nested includes //it matters, when the 'include' token is the last in previous stream (using super, lexer 'crashes' returning EOF token) token = this.nextToken(); } // Skip first token after switching on another input. // You need to use this rather than super as there may be nested include files if(((CommonToken)token).getStartIndex() < 0) { token = this.nextToken(); } return token; }
/** The standard method called to automatically emit a token at the * outermost lexical rule. The token object should point into the * char buffer start..stop. If there is a text override in 'text', * use that to set the token's text. Override this method to emit * custom Token objects. * * If you are building trees, then you should also override * Parser or TreeParser.getMissingSymbol(). */ public Token emit() { Token t = new CommonToken(input, state.type, state.channel, state.tokenStartCharIndex, getCharIndex()-1); t.setLine(state.tokenStartLine); t.setText(state.text); t.setCharPositionInLine(state.tokenStartCharPositionInLine); emit(t); return t; }
public void reportError(RecognitionException e) { /** TODO: not thought about recovery in lexer yet. * // if we've already reported an error and have not matched a token // yet successfully, don't report any errors. if ( errorRecovery ) { //System.err.print("[SPURIOUS] "); return; } errorRecovery = true; */ displayRecognitionError(this.getTokenNames(), e); }
/** Set the char stream and reset the lexer */ public void setCharStream(CharStream input) { this.input = null; reset(); this.input = input; }
@Override public void reportError(RecognitionException e) { if (throwExceptionOnError) { throw new IllegalArgumentException(e); } else { super.reportError(e); } }
if ( e instanceof MismatchedTokenException ) { MismatchedTokenException mte = (MismatchedTokenException)e; msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting "+getCharErrorDisplay(mte.expecting); msg = "no viable alternative at character "+getCharErrorDisplay(e.c); msg = "required (...)+ loop did not match anything at character "+getCharErrorDisplay(e.c); msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting set "+mse.expecting; msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting set "+mse.expecting; msg = "mismatched character "+getCharErrorDisplay(e.c)+" expecting set "+ getCharErrorDisplay(mre.a)+".."+getCharErrorDisplay(mre.b);
@Override public void recover(RecognitionException e) { super.recover(e); }
public void match(int c) throws MismatchedTokenException { if ( input.LA(1)!=c ) { if ( state.backtracking>0 ) { state.failed = true; return; } MismatchedTokenException mte = new MismatchedTokenException(c, input); recover(mte); // don't really recover; just consume in lexer throw mte; } input.consume(); state.failed = false; }
state.token = null; state.channel = Token.DEFAULT_CHANNEL; state.tokenStartCharIndex = input.index(); state.tokenStartCharPositionInLine = input.getCharPositionInLine(); state.tokenStartLine = input.getLine(); state.text = null; if ( input.LA(1)==CharStream.EOF ) { return getEOFToken(); mTokens(); if ( state.token==null ) { emit(); reportError(re); reportError(re); reportError(re); recover(re); // throw out current char and try again
public Token nextToken(){ //The following code was pulled out from super.nextToken() if (input.index() == 0) { try { state.token = null; state.channel = Token.DEFAULT_CHANNEL; state.tokenStartCharIndex = input.index(); state.tokenStartCharPositionInLine = input.getCharPositionInLine(); state.tokenStartLine = input.getLine(); state.text = null; mFirstBodyString(); state.type = BodyString; emit(); return state.token; } catch (NoViableAltException nva) { reportError(nva); recover(nva); // throw out current char and try again } catch (RecognitionException re) { reportError(re); // match() routine has already called recover() } } return super.nextToken(); }
/** The standard method called to automatically emit a token at the * outermost lexical rule. The token object should point into the * char buffer start..stop. If there is a text override in 'text', * use that to set the token's text. Override this method to emit * custom Token objects. * * If you are building trees, then you should also override * Parser or TreeParser.getMissingSymbol(). */ public Token emit() { Token t = new CommonToken(input, state.type, state.channel, state.tokenStartCharIndex, getCharIndex()-1); t.setLine(state.tokenStartLine); t.setText(state.text); t.setCharPositionInLine(state.tokenStartCharPositionInLine); emit(t); return t; }
@Override public void reportError(RecognitionException e) { /** TODO: not thought about recovery in lexer yet. * // if we've already reported an error and have not matched a token // yet successfully, don't report any errors. if ( errorRecovery ) { //System.err.print("[SPURIOUS] "); return; } errorRecovery = true; */ displayRecognitionError(this.getTokenNames(), e); }