resetTokenList(); addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); backslash = false; addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); backslash = false; addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); backslash = false; addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); currentTokenType = Token.NULL; backslash = !backslash; addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); backslash = false; addToken(text, currentTokenStart,i, Token.OPERATOR, newStartOffset+currentTokenStart); currentTokenType = Token.NULL; break; addToken(text, currentTokenStart,i, Token.SEPARATOR, newStartOffset+currentTokenStart); currentTokenType = Token.NULL; break; addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); currentTokenType = Token.NULL; break;
@Test public void testGetTokenList_nullStart_LiteralBackQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "`cat foo.txt`"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.LITERAL_BACKQUOTE, text)); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetLineCommentStartAndEnd() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String[] startAndEnd = tm.getLineCommentStartAndEnd(0); Assert.assertEquals("#", startAndEnd[0]); Assert.assertEquals(null, startAndEnd[1]); }
@Test public void testGetMarkOccurrencesOfTokenType() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); for (int i=0; i<TokenTypes.DEFAULT_NUM_TOKEN_TYPES; i++) { boolean expected = i == TokenTypes.IDENTIFIER || i == TokenTypes.VARIABLE; Assert.assertEquals(expected, tm.getMarkOccurrencesOfTokenType(i)); } }
/** * Overridden to cache the last token lexeme and type passed into this * method. */ @Override public void addToken(char[] array, int start, int end, int tokenType, int startOffset, boolean hyperlink) { this.lastTokenLexeme = new String(array, start, end-start+1); this.lastTokenType = tokenType; super.addToken(array, start, end, tokenType, startOffset, hyperlink); }
@Test public void testGetWordsToHighlight() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); TokenMap tokens = tm.getWordsToHighlight();
resetTokenList(); addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); backslash = false; addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); backslash = false; addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); backslash = false; addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); currentTokenType = Token.NULL; backslash = !backslash; addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); backslash = false; addToken(text, currentTokenStart,i, Token.OPERATOR, newStartOffset+currentTokenStart); currentTokenType = Token.NULL; break; addToken(text, currentTokenStart,i, Token.SEPARATOR, newStartOffset+currentTokenStart); currentTokenType = Token.NULL; break; addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); currentTokenType = Token.NULL; break;
@Test public void testGetTokenList_nullStart_EolComment() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "# This is a comment"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue("Unexpected token type: " + token, token.is(TokenTypes.COMMENT_EOL, text)); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
resetTokenList(); addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); backslash = false; addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); backslash = false; addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); backslash = false; addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); currentTokenType = Token.NULL; backslash = !backslash; addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); backslash = false; addToken(text, currentTokenStart,i, Token.OPERATOR, newStartOffset+currentTokenStart); currentTokenType = Token.NULL; break; addToken(text, currentTokenStart,i, Token.SEPARATOR, newStartOffset+currentTokenStart); currentTokenType = Token.NULL; break; addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); currentTokenType = Token.NULL; break;
@Test public void testGetTokenList_nullStart_LiteralDoubleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "\"Hello world\""; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.LITERAL_STRING_DOUBLE_QUOTE, text)); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
resetTokenList(); addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); backslash = false; addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); backslash = false; addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); backslash = false; addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); currentTokenType = Token.NULL; backslash = !backslash; addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); backslash = false; addToken(text, currentTokenStart,i, Token.OPERATOR, newStartOffset+currentTokenStart); currentTokenType = Token.NULL; break; addToken(text, currentTokenStart,i, Token.SEPARATOR, newStartOffset+currentTokenStart); currentTokenType = Token.NULL; break; addToken(text, currentTokenStart,i, Token.IDENTIFIER, newStartOffset+currentTokenStart); currentTokenType = Token.NULL; break;
@Test public void testGetTokenList_nullStart_LiteralSingleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "'Hello world'"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.LITERAL_CHAR, text)); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_nullStart_WhiteSpace() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " "; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, text)); text = "\t"; s = createSegment(text); token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, text)); }
@Test public void testGetTokenList_whitespaceStart_LiteralDoubleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " \"Hello world\""; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_STRING_DOUBLE_QUOTE, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_whitespaceStart_LiteralSingleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " 'Hello world'"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_CHAR, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_whitespaceStart_Variable() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " $PATH"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.VARIABLE, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_whitespaceStart_LiteralBackQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " `cat foo.txt`"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_BACKQUOTE, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_whitespaceStart_EolComment() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " # This is a comment"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue("Unexpected token type: " + token, token.is(TokenTypes.COMMENT_EOL, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_identifierStart_LiteralDoubleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "foo\"Hello world\""; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.IDENTIFIER, "foo")); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_STRING_DOUBLE_QUOTE, "\"Hello world\"")); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_identifierStart_LiteralBackQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "foo`cat foo.txt`"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.IDENTIFIER, "foo")); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_BACKQUOTE, "`cat foo.txt`")); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }