/** * Returns a {@link TokenMaker} for the specified key. * * @param key The key. * @return The corresponding <code>TokenMaker</code>, or * {@link PlainTextTokenMaker} if none matches the specified key. */ public final TokenMaker getTokenMaker(String key) { TokenMaker tm = getTokenMakerImpl(key); if (tm==null) { tm = new PlainTextTokenMaker(); } return tm; }
/** * Returns a {@link TokenMaker} for the specified key. * * @param key The key. * @return The corresponding <code>TokenMaker</code>, or * {@link PlainTextTokenMaker} if none matches the specified key. */ public final TokenMaker getTokenMaker(String key) { TokenMaker tm = getTokenMakerImpl(key); if (tm==null) { tm = new PlainTextTokenMaker(); } return tm; }
/** * Returns a {@link TokenMaker} for the specified key. * * @param key The key. * @return The corresponding <code>TokenMaker</code>, or * {@link PlainTextTokenMaker} if none matches the specified key. */ public final TokenMaker getTokenMaker(String key) { TokenMaker tm = getTokenMakerImpl(key); if (tm==null) { tm = new PlainTextTokenMaker(); } return tm; }
/** * Returns a {@link TokenMaker} for the specified key. * * @param key The key. * @return The corresponding <code>TokenMaker</code>, or * {@link PlainTextTokenMaker} if none matches the specified key. */ public final TokenMaker getTokenMaker(String key) { TokenMaker tm = getTokenMakerImpl(key); if (tm==null) { tm = new PlainTextTokenMaker(); } return tm; }
@Test public void testWhitespace() { String code = " foo bar\t\tbas\t \tbaz "; PlainTextTokenMaker tm = new PlainTextTokenMaker(); Segment segment = createSegment(code); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.isWhitespace()); token = token.getNextToken(); while (token != null && token.isPaintable()) { Assert.assertEquals("Not an identifier: " + token, TokenTypes.IDENTIFIER, token.getType()); token = token.getNextToken(); Assert.assertTrue(token.isWhitespace()); token = token.getNextToken(); } }
@Test public void testIdentifiers() { String code = " foo bar\t\tbas\t \tbaz "; PlainTextTokenMaker tm = new PlainTextTokenMaker(); Segment segment = createSegment(code); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.isWhitespace()); token = token.getNextToken(); while (token != null && token.isPaintable()) { Assert.assertEquals("Not an identifier: " + token, TokenTypes.IDENTIFIER, token.getType()); token = token.getNextToken(); Assert.assertTrue(token.isWhitespace()); token = token.getNextToken(); } }
PlainTextTokenMaker tm = new PlainTextTokenMaker();