@Test public void testWhitespace() { String code = " foo bar\t\tbas\t \tbaz "; PlainTextTokenMaker tm = new PlainTextTokenMaker(); Segment segment = createSegment(code); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.isWhitespace()); token = token.getNextToken(); while (token != null && token.isPaintable()) { Assert.assertEquals("Not an identifier: " + token, TokenTypes.IDENTIFIER, token.getType()); token = token.getNextToken(); Assert.assertTrue(token.isWhitespace()); token = token.getNextToken(); } }
@Test public void testIdentifiers() { String code = " foo bar\t\tbas\t \tbaz "; PlainTextTokenMaker tm = new PlainTextTokenMaker(); Segment segment = createSegment(code); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.isWhitespace()); token = token.getNextToken(); while (token != null && token.isPaintable()) { Assert.assertEquals("Not an identifier: " + token, TokenTypes.IDENTIFIER, token.getType()); token = token.getNextToken(); Assert.assertTrue(token.isWhitespace()); token = token.getNextToken(); } }
Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.isHyperlink()); Assert.assertEquals(TokenTypes.IDENTIFIER, token.getType());