/** * Returns a {@link TokenMaker} for the specified key. * * @param key The key. * @return The corresponding <code>TokenMaker</code>, or * {@link PlainTextTokenMaker} if none matches the specified key. */ public final TokenMaker getTokenMaker(String key) { TokenMaker tm = getTokenMakerImpl(key); if (tm==null) { tm = new PlainTextTokenMaker(); } return tm; }
boolean eof = zzRefill(); { addToken(Token.WHITESPACE, false); { addNullToken(); return firstToken; { addToken(Token.IDENTIFIER, true); { addToken(Token.IDENTIFIER, false); switch (zzLexicalState) { case YYINITIAL: { addNullToken(); return firstToken; zzScanError(ZZ_NO_MATCH);
/** * Returns the first token in the linked list of tokens generated * from <code>text</code>. This method must be implemented by * subclasses so they can correctly implement syntax highlighting. * * @param text The text from which to get tokens. * @param initialTokenType The token type we should start with. * @param startOffset The offset into the document at which * <code>text</code> starts. * @return The first <code>Token</code> in a linked list representing * the syntax highlighted text. */ @Override public Token getTokenList(Segment text, int initialTokenType, int startOffset) { resetTokenList(); this.offsetShift = -text.offset + startOffset; // Start off in the proper state. s = text; try { yyreset(zzReader); yybegin(YYINITIAL); return yylex(); } catch (IOException ioe) { ioe.printStackTrace(); return new TokenImpl(); } }
/** * Pushes the specified amount of characters back into the input stream. * * They will be read again by then next call of the scanning method * * @param number the number of characters to be read again. * This number must not be greater than yylength()! */ public void yypushback(int number) { if ( number > yylength() ) zzScanError(ZZ_PUSHBACK_2BIG); zzMarkedPos -= number; }
@Test public void testIdentifiers() { String code = " foo bar\t\tbas\t \tbaz "; PlainTextTokenMaker tm = new PlainTextTokenMaker(); Segment segment = createSegment(code); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.isWhitespace()); token = token.getNextToken(); while (token != null && token.isPaintable()) { Assert.assertEquals("Not an identifier: " + token, TokenTypes.IDENTIFIER, token.getType()); token = token.getNextToken(); Assert.assertTrue(token.isWhitespace()); token = token.getNextToken(); } }
/** * Pushes the specified amount of characters back into the input stream. * * They will be read again by then next call of the scanning method * * @param number the number of characters to be read again. * This number must not be greater than yylength()! */ public void yypushback(int number) { if ( number > yylength() ) zzScanError(ZZ_PUSHBACK_2BIG); zzMarkedPos -= number; }
@Test public void testWhitespace() { String code = " foo bar\t\tbas\t \tbaz "; PlainTextTokenMaker tm = new PlainTextTokenMaker(); Segment segment = createSegment(code); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.isWhitespace()); token = token.getNextToken(); while (token != null && token.isPaintable()) { Assert.assertEquals("Not an identifier: " + token, TokenTypes.IDENTIFIER, token.getType()); token = token.getNextToken(); Assert.assertTrue(token.isWhitespace()); token = token.getNextToken(); } }
/** * Returns the first token in the linked list of tokens generated * from <code>text</code>. This method must be implemented by * subclasses so they can correctly implement syntax highlighting. * * @param text The text from which to get tokens. * @param initialTokenType The token type we should start with. * @param startOffset The offset into the document at which * <code>text</code> starts. * @return The first <code>Token</code> in a linked list representing * the syntax highlighted text. */ public Token getTokenList(Segment text, int initialTokenType, int startOffset) { resetTokenList(); this.offsetShift = -text.offset + startOffset; // Start off in the proper state. s = text; try { yyreset(zzReader); yybegin(YYINITIAL); return yylex(); } catch (IOException ioe) { ioe.printStackTrace(); return new DefaultToken(); } }
boolean eof = zzRefill(); { addToken(Token.WHITESPACE, false); { addNullToken(); return firstToken; { addToken(Token.IDENTIFIER, true); { addToken(Token.IDENTIFIER, false); switch (zzLexicalState) { case YYINITIAL: { addNullToken(); return firstToken; zzScanError(ZZ_NO_MATCH);
/** * Pushes the specified amount of characters back into the input stream. * * They will be read again by then next call of the scanning method * * @param number the number of characters to be read again. * This number must not be greater than yylength()! */ public void yypushback(int number) { if ( number > yylength() ) zzScanError(ZZ_PUSHBACK_2BIG); zzMarkedPos -= number; }
PlainTextTokenMaker tm = new PlainTextTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.isHyperlink()); Assert.assertEquals(TokenTypes.IDENTIFIER, token.getType());
/** * Returns a {@link TokenMaker} for the specified key. * * @param key The key. * @return The corresponding <code>TokenMaker</code>, or * {@link PlainTextTokenMaker} if none matches the specified key. */ public final TokenMaker getTokenMaker(String key) { TokenMaker tm = getTokenMakerImpl(key); if (tm==null) { tm = new PlainTextTokenMaker(); } return tm; }
/** * Returns the first token in the linked list of tokens generated * from <code>text</code>. This method must be implemented by * subclasses so they can correctly implement syntax highlighting. * * @param text The text from which to get tokens. * @param initialTokenType The token type we should start with. * @param startOffset The offset into the document at which * <code>text</code> starts. * @return The first <code>Token</code> in a linked list representing * the syntax highlighted text. */ public Token getTokenList(Segment text, int initialTokenType, int startOffset) { resetTokenList(); this.offsetShift = -text.offset + startOffset; // Start off in the proper state. s = text; try { yyreset(zzReader); yybegin(YYINITIAL); return yylex(); } catch (IOException ioe) { ioe.printStackTrace(); return new DefaultToken(); } }
boolean eof = zzRefill(); { addToken(Token.WHITESPACE, false); { addNullToken(); return firstToken; { addToken(Token.IDENTIFIER, true); { addToken(Token.IDENTIFIER, false); switch (zzLexicalState) { case YYINITIAL: { addNullToken(); return firstToken; zzScanError(ZZ_NO_MATCH);
/** * Pushes the specified amount of characters back into the input stream. * * They will be read again by then next call of the scanning method * * @param number the number of characters to be read again. * This number must not be greater than yylength()! */ public void yypushback(int number) { if ( number > yylength() ) zzScanError(ZZ_PUSHBACK_2BIG); zzMarkedPos -= number; }
/** * Returns a {@link TokenMaker} for the specified key. * * @param key The key. * @return The corresponding <code>TokenMaker</code>, or * {@link PlainTextTokenMaker} if none matches the specified key. */ public final TokenMaker getTokenMaker(String key) { TokenMaker tm = getTokenMakerImpl(key); if (tm==null) { tm = new PlainTextTokenMaker(); } return tm; }
/** * Returns the first token in the linked list of tokens generated * from <code>text</code>. This method must be implemented by * subclasses so they can correctly implement syntax highlighting. * * @param text The text from which to get tokens. * @param initialTokenType The token type we should start with. * @param startOffset The offset into the document at which * <code>text</code> starts. * @return The first <code>Token</code> in a linked list representing * the syntax highlighted text. */ @Override public Token getTokenList(Segment text, int initialTokenType, int startOffset) { resetTokenList(); this.offsetShift = -text.offset + startOffset; // Start off in the proper state. s = text; try { yyreset(zzReader); yybegin(YYINITIAL); return yylex(); } catch (IOException ioe) { ioe.printStackTrace(); return new TokenImpl(); } }
boolean eof = zzRefill(); { addToken(Token.WHITESPACE, false); { addNullToken(); return firstToken; { addToken(Token.IDENTIFIER, true); { addToken(Token.IDENTIFIER, false); switch (zzLexicalState) { case YYINITIAL: { addNullToken(); return firstToken; zzScanError(ZZ_NO_MATCH);
/** * Returns a {@link TokenMaker} for the specified key. * * @param key The key. * @return The corresponding <code>TokenMaker</code>, or * {@link PlainTextTokenMaker} if none matches the specified key. */ public final TokenMaker getTokenMaker(String key) { TokenMaker tm = getTokenMakerImpl(key); if (tm==null) { tm = new PlainTextTokenMaker(); } return tm; }