@Test public void testGetTokenList_nullStart_WhiteSpace() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " "; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, text)); text = "\t"; s = createSegment(text); token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, text)); }
@Test public void testIdentifiers() { String code = " foo bar\t\tbas\t \tbaz "; PlainTextTokenMaker tm = new PlainTextTokenMaker(); Segment segment = createSegment(code); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.isWhitespace()); token = token.getNextToken(); while (token != null && token.isPaintable()) { Assert.assertEquals("Not an identifier: " + token, TokenTypes.IDENTIFIER, token.getType()); token = token.getNextToken(); Assert.assertTrue(token.isWhitespace()); token = token.getNextToken(); } }
@Test public void testWhitespace() { String code = " foo bar\t\tbas\t \tbaz "; PlainTextTokenMaker tm = new PlainTextTokenMaker(); Segment segment = createSegment(code); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.isWhitespace()); token = token.getNextToken(); while (token != null && token.isPaintable()) { Assert.assertEquals("Not an identifier: " + token, TokenTypes.IDENTIFIER, token.getType()); token = token.getNextToken(); Assert.assertTrue(token.isWhitespace()); token = token.getNextToken(); } }
@Test public void testJava_Annotations() { String code = "@Test @Foo @Foo_Bar_Bas @Number7"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, JSPTokenMaker.INTERNAL_IN_JAVA_EXPRESSION, 0); String[] keywords = code.split(" +"); for (int i = 0; i < keywords.length; i++) { Assert.assertEquals(keywords[i], token.getLexeme()); Assert.assertEquals("Unexpected token type for token: " + token, TokenTypes.ANNOTATION, token.getType()); if (i < keywords.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } }
@Test public void testJS_DataTypes() { String code = "boolean byte char double float int long short"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, JS_PREV_TOKEN_TYPE, 0); String[] keywords = code.split(" +"); for (int i = 0; i < keywords.length; i++) { Assert.assertEquals(keywords[i], token.getLexeme()); Assert.assertEquals(TokenTypes.DATA_TYPE, token.getType()); if (i < keywords.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } }
@Test public void testJS_Functions() { String code = "eval parseInt parseFloat escape unescape isNaN isFinite"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, HTMLTokenMaker.INTERNAL_IN_JS, 0); String[] functions = code.split(" +"); for (int i = 0; i < functions.length; i++) { Assert.assertEquals(functions[i], token.getLexeme()); Assert.assertEquals("Not a function token: " + token, TokenTypes.FUNCTION, token.getType()); if (i < functions.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } }
@Test public void testJava_BooleanLiterals() { String code = "true false"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, JSPTokenMaker.INTERNAL_IN_JAVA_EXPRESSION, 0); String[] keywords = code.split(" +"); for (int i = 0; i < keywords.length; i++) { Assert.assertEquals(keywords[i], token.getLexeme()); Assert.assertEquals(TokenTypes.LITERAL_BOOLEAN, token.getType()); if (i < keywords.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } }
@Test public void testJava_DataTypes() { String code = "boolean byte char double float int long short"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, JSPTokenMaker.INTERNAL_IN_JAVA_EXPRESSION, 0); String[] keywords = code.split(" +"); for (int i = 0; i < keywords.length; i++) { Assert.assertEquals(keywords[i], token.getLexeme()); Assert.assertEquals(TokenTypes.DATA_TYPE, token.getType()); if (i < keywords.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } }
@Test public void testJS_Functions() { String code = "eval parseInt parseFloat escape unescape isNaN isFinite"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, JS_PREV_TOKEN_TYPE, 0); String[] functions = code.split(" +"); for (int i = 0; i < functions.length; i++) { Assert.assertEquals(functions[i], token.getLexeme()); Assert.assertEquals("Not a function token: " + token, TokenTypes.FUNCTION, token.getType()); if (i < functions.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } }
@Test public void testJS_BooleanLiterals() { String code = "true false"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, JS_PREV_TOKEN_TYPE, 0); String[] keywords = code.split(" +"); for (int i = 0; i < keywords.length; i++) { Assert.assertEquals(keywords[i], token.getLexeme()); Assert.assertEquals(TokenTypes.LITERAL_BOOLEAN, token.getType()); if (i < keywords.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } }
@Test public void testJS_BooleanLiterals() { String code = "true false"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, HTMLTokenMaker.INTERNAL_IN_JS, 0); String[] keywords = code.split(" +"); for (int i = 0; i < keywords.length; i++) { Assert.assertEquals(keywords[i], token.getLexeme()); Assert.assertEquals(TokenTypes.LITERAL_BOOLEAN, token.getType()); if (i < keywords.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } }
@Test public void testJS_DataTypes() { String code = "boolean byte char double float int long short"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, HTMLTokenMaker.INTERNAL_IN_JS, 0); String[] keywords = code.split(" +"); for (int i = 0; i < keywords.length; i++) { Assert.assertEquals(keywords[i], token.getLexeme()); Assert.assertEquals(TokenTypes.DATA_TYPE, token.getType()); if (i < keywords.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } }
@Test public void testJS_Functions() { String code = "eval parseInt parseFloat escape unescape isNaN isFinite"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, JS_PREV_TOKEN_TYPE, 0); String[] functions = code.split(" +"); for (int i = 0; i < functions.length; i++) { Assert.assertEquals(functions[i], token.getLexeme()); Assert.assertEquals("Not a function token: " + token, TokenTypes.FUNCTION, token.getType()); if (i < functions.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } }
@Test public void testJS_BooleanLiterals() { String code = "true false"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, JS_PREV_TOKEN_TYPE, 0); String[] keywords = code.split(" +"); for (int i = 0; i < keywords.length; i++) { Assert.assertEquals(keywords[i], token.getLexeme()); Assert.assertEquals(TokenTypes.LITERAL_BOOLEAN, token.getType()); if (i < keywords.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } }
@Test public void testJS_DataTypes() { String code = "boolean byte char double float int long short"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, JS_PREV_TOKEN_TYPE, 0); String[] keywords = code.split(" +"); for (int i = 0; i < keywords.length; i++) { Assert.assertEquals(keywords[i], token.getLexeme()); Assert.assertEquals(TokenTypes.DATA_TYPE, token.getType()); if (i < keywords.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } }
@Test public void testJS_BooleanLiterals() { String code = "true false"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); String[] keywords = code.split(" +"); for (int i = 0; i < keywords.length; i++) { Assert.assertEquals(keywords[i], token.getLexeme()); Assert.assertEquals(TokenTypes.LITERAL_BOOLEAN, token.getType()); if (i < keywords.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } Assert.assertTrue(token.getType() == TokenTypes.NULL); }
@Test public void testJS_DataTypes() { String code = "String Number int uint Boolean Null"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); String[] keywords = code.split(" +"); for (int i = 0; i < keywords.length; i++) { Assert.assertEquals(keywords[i], token.getLexeme()); Assert.assertEquals(TokenTypes.DATA_TYPE, token.getType()); if (i < keywords.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } Assert.assertTrue(token.getType() == TokenTypes.NULL); }
@Test public void testIntegerLiterals() { String code = "1 42 0 -1 -42"; Segment segment = createSegment(code); JshintrcTokenMaker tm = new JshintrcTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); String[] keywords = code.split(" +"); for (int i = 0; i < keywords.length; i++) { Assert.assertEquals("Unexpected number for token " + i, keywords[i], token.getLexeme()); Assert.assertEquals(TokenTypes.LITERAL_NUMBER_DECIMAL_INT, token.getType()); if (i < keywords.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } Assert.assertTrue(token.getType() == TokenTypes.NULL); }
@Test public void testGetTokenList_nullStart_Operators() { String code = "= | > < &"; Segment segment = createSegment(code); UnixShellTokenMaker tm = new UnixShellTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); String[] keywords = code.split(" +"); for (int i = 0; i < keywords.length; i++) { Assert.assertEquals(keywords[i], token.getLexeme()); Assert.assertEquals(TokenTypes.OPERATOR, token.getType()); if (i < keywords.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue("Not a single space: " + token, token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } Assert.assertTrue(token.getType() == TokenTypes.NULL); }
@Test public void testIntegerLiterals() { String code = "1 42 0 -1 -42"; Segment segment = createSegment(code); JsonTokenMaker tm = new JsonTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); String[] keywords = code.split(" +"); for (int i = 0; i < keywords.length; i++) { Assert.assertEquals("Unexpected number for token " + i, keywords[i], token.getLexeme()); Assert.assertEquals(TokenTypes.LITERAL_NUMBER_DECIMAL_INT, token.getType()); if (i < keywords.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } Assert.assertTrue(token.getType() == TokenTypes.NULL); }