@Nullable private Token concatTokensUntil(Token start, String endText) { StringBuilder sb = new StringBuilder(); Token current = start; while (current != null && current.getType() != TokenTypes.NULL) { String text = current.getLexeme(); if (text != null) { sb.append(text); if (text.equals(endText)) { char[] line = sb.toString().toCharArray(); TokenImpl token = new TokenImpl(line, 0, line.length - 1, start.getOffset(), start.getType(), start.getLanguageIndex()); token.setNextToken(current.getNextToken()); return token; } } current = current.getNextToken(); } return null; } }
private void processTokens(Token tokens) { Token prev = null; Token current = tokens; while (current != null) { if (prev != null) { int tokenType = current.getType(); if (tokenType == TokenTypes.IDENTIFIER) { current = mergeLongClassNames(prev, current, false); } else if (tokenType == TokenTypes.ANNOTATION) { current = mergeLongClassNames(prev, current, true); } } prev = current; current = current.getNextToken(); } }
/** * {@inheritDoc} */ @Override public int getLastTokenTypeOnLine(Segment text, int initialTokenType) { // Last parameter doesn't matter if we're not painting. Token t = getTokenList(text, initialTokenType, 0); while (t.getNextToken()!=null) { t = t.getNextToken(); } return t.getType(); }
@Override public Token getLastPaintableToken() { Token t = this; while (t.isPaintable()) { Token next = t.getNextToken(); if (next==null || !next.isPaintable()) { return t; } t = next; } return null; }
@Test public void testGetTokenList_whitespaceStart_LiteralDoubleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " \"Hello world\""; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_STRING_DOUBLE_QUOTE, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_whitespaceStart_LiteralSingleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " 'Hello world'"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_CHAR, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_whitespaceStart_Variable() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " $PATH"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.VARIABLE, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_whitespaceStart_LiteralBackQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " `cat foo.txt`"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_BACKQUOTE, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testKeywords() { String code = "127.0.0.1 localhost"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.RESERVED_WORD, "127.0.0.1")); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " ")); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.IDENTIFIER, "localhost")); }
@Test public void testNameValuePair_happyPath() { String code = "dialog.title=Options"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.DATA_TYPE, "dialog.title")); token = token.getNextToken(); Assert.assertTrue(token.isSingleChar(TokenTypes.OPERATOR, '=')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.IDENTIFIER, "Options")); }
@Test public void testGetTokenList_identifierStart_LiteralDoubleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "foo\"Hello world\""; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.IDENTIFIER, "foo")); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_STRING_DOUBLE_QUOTE, "\"Hello world\"")); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testHtml_happyPath_closingTag() { String code = "</body>"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.MARKUP_TAG_DELIMITER, "</")); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.MARKUP_TAG_NAME, "body")); token = token.getNextToken(); Assert.assertTrue(token.isSingleChar(TokenTypes.MARKUP_TAG_DELIMITER, '>')); }
@Test public void testGetTokenList_identifierStart_LiteralBackQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "foo`cat foo.txt`"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.IDENTIFIER, "foo")); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_BACKQUOTE, "`cat foo.txt`")); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_identifierStart_LiteralSingleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "foo'Hello world'"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.IDENTIFIER, "foo")); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_CHAR, "'Hello world'")); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testJS_DocComments_Markup() { String text = "<code>"; Segment segment = createSegment(text); TokenMaker tm = createTokenMaker(); final int INTERNAL_IN_JS_COMMENT_DOCUMENTATION = -9; Token token = tm.getTokenList(segment, INTERNAL_IN_JS_COMMENT_DOCUMENTATION, 0); // Can sometimes produce empty tokens, if e.g. @foo is first token // on a line. We could technically make that better, but it is not // the common case token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.COMMENT_MARKUP, "<code>")); }
@Test public void testEolComments_escapedPercentNotAComment() { String code = "\\% not-comment"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.SEPARATOR, '\\')); token = token.getNextToken(); Assert.assertTrue(token.isSingleChar(TokenTypes.IDENTIFIER, '%')); }
@Test public void testGetTokenList_nullStart_LiteralBackQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "`cat foo.txt`"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.LITERAL_BACKQUOTE, text)); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_nullStart_EolComment() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "# This is a comment"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue("Unexpected token type: " + token, token.is(TokenTypes.COMMENT_EOL, text)); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_nullStart_LiteralDoubleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "\"Hello world\""; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.LITERAL_STRING_DOUBLE_QUOTE, text)); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_nullStart_LiteralSingleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "'Hello world'"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.LITERAL_CHAR, text)); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }