protected Token outside() { if ( input.getCharPositionInLine()==0 && (c==' '||c=='\t') ) { while ( c==' ' || c=='\t' ) consume(); // scarf indent if ( c!=EOF ) return newToken(INDENT); return newToken(TEXT); } if ( c==delimiterStartChar ) { consume(); if ( c=='!' ) return COMMENT(); if ( c=='\\' ) return ESCAPE(); // <\\> <\uFFFF> <\n> etc... scanningInsideExpr = true; return newToken(LDELIM); } if ( c=='\r' ) { consume(); consume(); return newToken(NEWLINE); } // \r\n -> \n if ( c=='\n') { consume(); return newToken(NEWLINE); } if ( c=='}' && subtemplateDepth>0 ) { scanningInsideExpr = true; subtemplateDepth--; consume(); return newTokenFromPreviousChar(RCURLY); } return mTEXT(); }
Token ESCAPE() { startCharIndex = input.index(); startCharPositionInLine = input.getCharPositionInLine(); consume(); // kill \\ if ( c=='u') return UNICODE(); String text; switch ( c ) { case '\\' : LINEBREAK(); return SKIP; case 'n' : text = "\n"; break; case 't' : text = "\t"; break; case ' ' : text = " "; break; default : NoViableAltException e = new NoViableAltException("",0,0,input); errMgr.lexerError(input.getSourceName(), "invalid escaped char: '"+str(c)+"'", templateToken, e); consume(); match(delimiterStopChar); return SKIP; } consume(); Token t = newToken(TEXT, text, input.getCharPositionInLine()-2); match(delimiterStopChar); return t; }
int curlyPos = startCharPositionInLine; List<Token> argTokens = new ArrayList<Token>(); consume(); Token curly = newTokenFromPreviousChar(LCURLY); WS(); argTokens.add( mID() ); WS(); while ( c==',' ) { consume(); argTokens.add( newTokenFromPreviousChar(COMMA) ); WS(); argTokens.add( mID() ); WS(); WS(); if ( c=='|' ) { consume(); argTokens.add( newTokenFromPreviousChar(PIPE) ); if ( isWS(c) ) consume(); // ignore a single whitespace after | for (Token t : argTokens) emit(t); input.release(m); scanningInsideExpr = false; consume(); scanningInsideExpr = false; return curly;
/** ID : ('a'..'z'|'A'..'Z'|'_'|'/') ('a'..'z'|'A'..'Z'|'0'..'9'|'_'|'/')* ; */ Token mID() { // called from subTemplate; so keep resetting position during speculation startCharIndex = input.index(); startLine = input.getLine(); startCharPositionInLine = input.getCharPositionInLine(); consume(); while ( isIDLetter(c) ) { consume(); } return newToken(ID); }
public Token _nextToken() { //System.out.println("nextToken: c="+(char)c+"@"+input.index()); while ( true ) { // lets us avoid recursion when skipping stuff startCharIndex = input.index(); startLine = input.getLine(); startCharPositionInLine = input.getCharPositionInLine(); if ( c==EOF ) return newToken(EOF_TYPE); Token t; if ( scanningInsideExpr ) t = inside(); else t = outside(); if ( t!=SKIP ) return t; } }
Token COMMENT() { match('!'); while ( !(c=='!' && input.LA(2)==delimiterStopChar) ) { if (c==EOF) { RecognitionException re = new MismatchedTokenException((int)'!', input); re.line = input.getLine(); re.charPositionInLine = input.getCharPositionInLine(); errMgr.lexerError(input.getSourceName(), "Nonterminated comment starting at " + startLine+":"+startCharPositionInLine+": '!"+ delimiterStopChar+"' missing", templateToken, re); break; } consume(); } consume(); consume(); // grab !> return newToken(COMMENT); }
Token UNICODE() { consume(); char[] chars = new char[4]; if ( !isUnicodeLetter(c) ) { NoViableAltException e = new NoViableAltException("",0,0,input); errMgr.lexerError(input.getSourceName(), "invalid unicode char: '"+str(c)+"'", templateToken, e); consume(); if ( !isUnicodeLetter(c) ) { NoViableAltException e = new NoViableAltException("",0,0,input); errMgr.lexerError(input.getSourceName(), "invalid unicode char: '"+str(c)+"'", templateToken, e); consume(); if ( !isUnicodeLetter(c) ) { NoViableAltException e = new NoViableAltException("",0,0,input); errMgr.lexerError(input.getSourceName(), "invalid unicode char: '"+str(c)+"'", templateToken, e); consume(); if ( !isUnicodeLetter(c) ) { NoViableAltException e = new NoViableAltException("",0,0,input); errMgr.lexerError(input.getSourceName(), "invalid unicode char: '"+str(c)+"'", templateToken, e); Token t = newToken(TEXT, String.valueOf(uc), input.getCharPositionInLine()-6); consume(); match(delimiterStopChar); return t;
public Token nextToken() { Token t = null; if ( tokens.size()>0 ) { t = tokens.remove(0); } else t = _nextToken(); //System.out.println(t); return t; }
void WS() { while ( c==' ' || c=='\t' || c=='\n' || c=='\r' ) consume(); }
/** <pre> * ID : ('a'..'z'|'A'..'Z'|'_'|'/') * ('a'..'z'|'A'..'Z'|'0'..'9'|'_'|'/')* * ; * </pre> */ Token mID() { // called from subTemplate; so keep resetting position during speculation startCharIndex = input.index(); startLine = input.getLine(); startCharPositionInLine = input.getCharPositionInLine(); consume(); while ( isIDLetter(c) ) { consume(); } return newToken(ID); }
public Token _nextToken() { //System.out.println("nextToken: c="+(char)c+"@"+input.index()); while ( true ) { // lets us avoid recursion when skipping stuff startCharIndex = input.index(); startLine = input.getLine(); startCharPositionInLine = input.getCharPositionInLine(); if ( c==EOF ) return newToken(EOF_TYPE); Token t; if ( scanningInsideExpr ) t = inside(); else t = outside(); if ( t!=SKIP ) return t; } }
Token COMMENT() { match('!'); while ( !(c=='!' && input.LA(2)==delimiterStopChar) ) { if (c==EOF) { RecognitionException re = new MismatchedTokenException((int)'!', input); re.line = input.getLine(); re.charPositionInLine = input.getCharPositionInLine(); errMgr.lexerError(input.getSourceName(), "Nonterminated comment starting at " + startLine+":"+startCharPositionInLine+": '!"+ delimiterStopChar+"' missing", templateToken, re); break; } consume(); } consume(); consume(); // grab !> return newToken(COMMENT); }
void LINEBREAK() { match('\\'); // only kill 2nd \ as ESCAPE() kills first one match(delimiterStopChar); while ( c==' ' || c=='\t' ) consume(); // scarf WS after <\\> if ( c==EOF ) { RecognitionException re = new RecognitionException(input); re.line = input.getLine(); re.charPositionInLine = input.getCharPositionInLine(); errMgr.lexerError(input.getSourceName(), "Missing newline after newline escape <\\\\>", templateToken, re); return; } if ( c=='\r' ) consume(); match('\n'); while ( c==' ' || c=='\t' ) consume(); // scarf any indent }