@Override public Token getTokenList(Segment text, int initialTokenType, int startOffset) { Token tokens = super.getTokenList(text, initialTokenType, startOffset); if (startOffset > 0 && tokens.getType() != TokenTypes.NULL) { try { processTokens(tokens); } catch (Exception e) { LOG.error("Process tokens failed for text: {}", text, e); } } return tokens; }
@Nullable private Token concatTokensUntil(Token start, String endText) { StringBuilder sb = new StringBuilder(); Token current = start; while (current != null && current.getType() != TokenTypes.NULL) { String text = current.getLexeme(); if (text != null) { sb.append(text); if (text.equals(endText)) { char[] line = sb.toString().toCharArray(); TokenImpl token = new TokenImpl(line, 0, line.length - 1, start.getOffset(), start.getType(), start.getLanguageIndex()); token.setNextToken(current.getNextToken()); return token; } } current = current.getNextToken(); } return null; } }
private void processTokens(Token tokens) { Token prev = null; Token current = tokens; while (current != null) { if (prev != null) { int tokenType = current.getType(); if (tokenType == TokenTypes.IDENTIFIER) { current = mergeLongClassNames(prev, current, false); } else if (tokenType == TokenTypes.ANNOTATION) { current = mergeLongClassNames(prev, current, true); } } prev = current; current = current.getNextToken(); } }
return null; int type = token.getType(); final int sourceOffset; if (type == TokenTypes.IDENTIFIER) {
/** * {@inheritDoc} */ @Override public boolean isValidType(RSyntaxTextArea textArea, Token t) { return textArea.getMarkOccurrencesOfTokenType(t.getType()); }
@Test public void testMultiLineComments() { String[] mlcLiterals = { "/* Hello world */", }; for (String code : mlcLiterals) { Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertEquals(TokenTypes.COMMENT_MULTILINE, token.getType()); } }
@Test public void testJS_DocComments() { String[] docCommentLiterals = { "/** Hello world */", }; for (String code : docCommentLiterals) { Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, JS_PREV_TOKEN_TYPE, 0); Assert.assertEquals(TokenTypes.COMMENT_DOCUMENTATION, token.getType()); } }
@Test public void testJS_EolComments() { String[] eolCommentLiterals = { "// Hello world", }; for (String code : eolCommentLiterals) { Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, JS_PREV_TOKEN_TYPE, 0); Assert.assertEquals(TokenTypes.COMMENT_EOL, token.getType()); } }
@Test public void testStringLiterals() { String[] stringLiterals = { "\"\"", "\"hi\"", "\"$\\\"escaped double quote\"", }; for (String code : stringLiterals) { Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertEquals("Not a string: " + token, TokenTypes.LITERAL_STRING_DOUBLE_QUOTE, token.getType()); } }
@Test public void testEolComments() { String[] eolCommentLiterals = { "# Hello world", }; for (String code : eolCommentLiterals) { Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertEquals(TokenTypes.COMMENT_EOL, token.getType()); } }
@Test public void testJava_DocComments() { String[] docCommentLiterals = { "/** Hello world */", }; for (String code : docCommentLiterals) { Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, JSPTokenMaker.INTERNAL_IN_JAVA_EXPRESSION, 0); Assert.assertEquals(TokenTypes.COMMENT_DOCUMENTATION, token.getType()); } }
@Test public void testJS_MultiLineComments() { String[] mlcLiterals = { "/* Hello world */", }; for (String code : mlcLiterals) { Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, JS_PREV_TOKEN_TYPE, 0); Assert.assertEquals(TokenTypes.COMMENT_MULTILINE, token.getType()); } }
@Test public void testDtd_comment() { String[] commentLiterals = { "<!-- Hello world -->", }; for (String code : commentLiterals) { Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertEquals(TokenTypes.MARKUP_COMMENT, token.getType()); } }
@Test public void testCss_comment() { String[] commentLiterals = { "/* Hello world */", }; for (String code : commentLiterals) { Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, CSS_PREV_TOKEN_TYPE, 0); Assert.assertEquals(TokenTypes.COMMENT_MULTILINE, token.getType()); } }
@Test public void testJS_DocComments() { String[] docCommentLiterals = { "/** Hello world */", }; for (String code : docCommentLiterals) { Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, JS_PREV_TOKEN_TYPE, 0); Assert.assertEquals(TokenTypes.COMMENT_DOCUMENTATION, token.getType()); } }
@Test public void testJS_Whitespace() { String[] whitespace = { " ", "\t", "\f", " \t ", }; for (String code : whitespace) { Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, JS_PREV_TOKEN_TYPE, 0); Assert.assertEquals(TokenTypes.WHITESPACE, token.getType()); } }
@Test public void testStringLiterals() { String[] stringLiterals = { "\"\"", "\"hi\"", "\"\\u00fe\"", "\"\\\"\"", }; for (String code : stringLiterals) { Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertEquals(TokenTypes.LITERAL_STRING_DOUBLE_QUOTE, token.getType()); } }
@Test public void testMultiLineComments() { String[] mlcLiterals = { "/* Hello world */", }; for (String code : mlcLiterals) { Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertEquals(TokenTypes.COMMENT_MULTILINE, token.getType()); } }
@Test public void testEolComments() { String[] eolCommentLiterals = { "// Hello world", }; for (String code : eolCommentLiterals) { Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertEquals(TokenTypes.COMMENT_EOL, token.getType()); } }
@Test public void testStringLiterals() { String[] stringLiterals = { "''", "'hi'", }; for (String code : stringLiterals) { Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertEquals("Not a string: " + token, TokenTypes.LITERAL_STRING_DOUBLE_QUOTE, token.getType()); } }