@Test public void testGetTokenList_nullStart_WhiteSpace() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " "; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, text)); text = "\t"; s = createSegment(text); token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, text)); }
@Test public void testGetTokenList_whitespaceStart_LiteralDoubleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " \"Hello world\""; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_STRING_DOUBLE_QUOTE, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_whitespaceStart_LiteralSingleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " 'Hello world'"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_CHAR, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_whitespaceStart_Variable() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " $PATH"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.VARIABLE, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_whitespaceStart_LiteralBackQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " `cat foo.txt`"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_BACKQUOTE, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_whitespaceStart_EolComment() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " # This is a comment"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue("Unexpected token type: " + token, token.is(TokenTypes.COMMENT_EOL, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_nullStart_LiteralBackQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "`cat foo.txt`"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.LITERAL_BACKQUOTE, text)); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_nullStart_EolComment() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "# This is a comment"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue("Unexpected token type: " + token, token.is(TokenTypes.COMMENT_EOL, text)); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_nullStart_LiteralDoubleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "\"Hello world\""; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.LITERAL_STRING_DOUBLE_QUOTE, text)); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_nullStart_LiteralSingleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "'Hello world'"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.LITERAL_CHAR, text)); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_nullStart_Variable() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "$PATH"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.VARIABLE, text)); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); text = "${varName}"; s = createSegment(text); token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.VARIABLE, text)); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_identifierStart_LiteralDoubleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "foo\"Hello world\""; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.IDENTIFIER, "foo")); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_STRING_DOUBLE_QUOTE, "\"Hello world\"")); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_identifierStart_LiteralBackQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "foo`cat foo.txt`"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.IDENTIFIER, "foo")); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_BACKQUOTE, "`cat foo.txt`")); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_identifierStart_LiteralSingleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "foo'Hello world'"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.IDENTIFIER, "foo")); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_CHAR, "'Hello world'")); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_nullStart_EscapedDollarSign() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); // Parsed as two identifiers: '\\' and '$' String text = "\\$"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue("Unexpected token type: " + token, token.is(TokenTypes.IDENTIFIER, "\\")); token = token.getNextToken(); Assert.assertTrue("Unexpected token type: " + token, token.is(TokenTypes.IDENTIFIER, "$")); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_nullStart_EscapedDoubleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); // Parsed as two identifiers: '\\' and '"' String text = "\\\""; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue("Unexpected token type: " + token, token.is(TokenTypes.IDENTIFIER, "\\")); token = token.getNextToken(); Assert.assertTrue("Unexpected token type: " + token, token.is(TokenTypes.IDENTIFIER, "\"")); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_nullStart_EscapedSingleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); // Parsed as two identifiers: '\\' and "'" String text = "\\'"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue("Unexpected token type: " + token, token.is(TokenTypes.IDENTIFIER, "\\")); token = token.getNextToken(); Assert.assertTrue("Unexpected token type: " + token, token.is(TokenTypes.IDENTIFIER, "'")); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_nullStart_EscapedBackQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); // Parsed as two identifiers: '\\' and '`' String text = "\\`"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue("Unexpected token type: " + token, token.is(TokenTypes.IDENTIFIER, "\\")); token = token.getNextToken(); Assert.assertTrue("Unexpected token type: " + token, token.is(TokenTypes.IDENTIFIER, "`")); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_nullStart_Operators() { String code = "= | > < &"; Segment segment = createSegment(code); UnixShellTokenMaker tm = new UnixShellTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); String[] keywords = code.split(" +"); for (int i = 0; i < keywords.length; i++) { Assert.assertEquals(keywords[i], token.getLexeme()); Assert.assertEquals(TokenTypes.OPERATOR, token.getType()); if (i < keywords.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue("Not a single space: " + token, token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } Assert.assertTrue(token.getType() == TokenTypes.NULL); }
@Test public void testGetTokenList_nullStart_Numeric() { String code = "42 722"; Segment segment = createSegment(code); UnixShellTokenMaker tm = new UnixShellTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); String[] keywords = code.split(" +"); for (int i = 0; i < keywords.length; i++) { Assert.assertEquals(keywords[i], token.getLexeme()); Assert.assertEquals(TokenTypes.LITERAL_NUMBER_DECIMAL_INT, token.getType()); if (i < keywords.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue("Not a single space: " + token, token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } Assert.assertTrue(token.getType() == TokenTypes.NULL); }