@Nullable private Token concatTokensUntil(Token start, String endText) { StringBuilder sb = new StringBuilder(); Token current = start; while (current != null && current.getType() != TokenTypes.NULL) { String text = current.getLexeme(); if (text != null) { sb.append(text); if (text.equals(endText)) { char[] line = sb.toString().toCharArray(); TokenImpl token = new TokenImpl(line, 0, line.length - 1, start.getOffset(), start.getType(), start.getLanguageIndex()); token.setNextToken(current.getNextToken()); return token; } } current = current.getNextToken(); } return null; } }
/** * Returns the token at the specified index, or <code>null</code> if * the given offset isn't in this token list's range.<br> * Note that this method does NOT check to see if <code>tokenList</code> * is null; callers should check for themselves. * * @param tokenList The list of tokens in which to search. * @param offset The offset at which to get the token. * @return The token at <code>offset</code>, or <code>null</code> if * none of the tokens are at that offset. * @see #getTokenAtOffset(RSyntaxTextArea, int) * @see #getTokenAtOffset(RSyntaxDocument, int) */ public static Token getTokenAtOffset(Token tokenList, int offset) { for (Token t=tokenList; t!=null && t.isPaintable(); t=t.getNextToken()){ if (t.containsPosition(offset)) { return t; } } return null; }
/** * Returns whether a token is the left bracket token. * * @param t The token. * @return Whether the token is the left bracket token. * @see #isRightBracket(Token) */ private static boolean isLeftBracket(Token t) { return t.getType()==TokenTypes.SEPARATOR && t.isSingleChar('['); }
private void processTokens(Token tokens) { Token prev = null; Token current = tokens; while (current != null) { if (prev != null) { int tokenType = current.getType(); if (tokenType == TokenTypes.IDENTIFIER) { current = mergeLongClassNames(prev, current, false); } else if (tokenType == TokenTypes.ANNOTATION) { current = mergeLongClassNames(prev, current, true); } } prev = current; current = current.getNextToken(); } }
return null; int type = token.getType(); final int sourceOffset; if (type == TokenTypes.IDENTIFIER) { sourceOffset = token.getOffset(); } else if (type == TokenTypes.ANNOTATION && token.length() > 1) { sourceOffset = token.getOffset() + 1; } else { return null; if (token.length() == 1) { char ch = token.getTextArray()[token.getTextOffset()]; if (ch == '.' || ch == ',' || ch == ';') { return null;
@Test public void testJS_DataTypes() { String code = "boolean byte char double float int long short"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, JS_PREV_TOKEN_TYPE, 0); String[] keywords = code.split(" +"); for (int i = 0; i < keywords.length; i++) { Assert.assertEquals(keywords[i], token.getLexeme()); Assert.assertEquals(TokenTypes.DATA_TYPE, token.getType()); if (i < keywords.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } }
@Test public void testEolComments_URL() { String[] eolCommentLiterals = { "// Hello world http://www.sas.com", }; for (String code : eolCommentLiterals) { Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertEquals(TokenTypes.COMMENT_EOL, token.getType()); token = token.getNextToken(); Assert.assertTrue(token.isHyperlink()); Assert.assertEquals(TokenTypes.COMMENT_EOL, token.getType()); Assert.assertEquals("http://www.sas.com", token.getLexeme()); } }
@Test public void testTS_EolComments_URL() { String[] eolCommentLiterals = { // Note: The 0-length token at the end of the first example is a // minor bug/performance thing "// Hello world http://www.sas.com", "// Hello world http://www.sas.com extra", }; for (String code : eolCommentLiterals) { Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TS_PREV_TOKEN_TYPE, 0); Assert.assertEquals(TokenTypes.COMMENT_EOL, token.getType()); token = token.getNextToken(); Assert.assertTrue(token.isHyperlink()); Assert.assertEquals(TokenTypes.COMMENT_EOL, token.getType()); Assert.assertEquals("http://www.sas.com", token.getLexeme()); token = token.getNextToken(); // Note: The 0-length token at the end of the first example is a // minor bug/performance thing if (token != null && token.isPaintable() && token.length() > 0) { Assert.assertFalse(token.isHyperlink()); Assert.assertTrue(token.is(TokenTypes.COMMENT_EOL, " extra")); } } }
@Test public void testNameValuePair_happyPath_withWhitespace() { String code = "dialog.title = Options"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.DATA_TYPE, "dialog.title")); token = token.getNextToken(); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.isSingleChar(TokenTypes.OPERATOR, '=')); token = token.getNextToken(); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.IDENTIFIER, "Options")); }
/** * Makes one token point to the same text segment, and have the same value * as another token. * * @param t2 The token from which to copy. */ public void copyFrom(Token t2) { text = t2.getTextArray(); textOffset = t2.getTextOffset(); textCount = t2.length(); setOffset(t2.getOffset()); setType(t2.getType()); hyperlink = t2.isHyperlink(); languageIndex = t2.getLanguageIndex(); nextToken = t2.getNextToken(); }
@Test public void testWhitespace() { String code = " foo bar\t\tbas\t \tbaz "; PlainTextTokenMaker tm = new PlainTextTokenMaker(); Segment segment = createSegment(code); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.isWhitespace()); token = token.getNextToken(); while (token != null && token.isPaintable()) { Assert.assertEquals("Not an identifier: " + token, TokenTypes.IDENTIFIER, token.getType()); token = token.getNextToken(); Assert.assertTrue(token.isWhitespace()); token = token.getNextToken(); } }
@Test public void testKeywords() { String code = "127.0.0.1 localhost"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.RESERVED_WORD, "127.0.0.1")); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.WHITESPACE, " ")); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.IDENTIFIER, "localhost")); }
@Test public void testHtml_comment_URL() { String code = "<!-- Hello world http://www.google.com -->"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertFalse(token.isHyperlink()); Assert.assertTrue(token.is(TokenTypes.MARKUP_COMMENT, "<!-- Hello world ")); token = token.getNextToken(); Assert.assertTrue(token.isHyperlink()); Assert.assertTrue(token.is(TokenTypes.MARKUP_COMMENT, "http://www.google.com")); token = token.getNextToken(); Assert.assertFalse(token.isHyperlink()); Assert.assertTrue(token.is(TokenTypes.MARKUP_COMMENT, " -->")); }
@Override public boolean equals(Object obj) { if (obj==this) { return true; } if (!(obj instanceof Token)) { return false; } Token t2 = (Token)obj; return offset==t2.getOffset() && type==t2.getType() && languageIndex==t2.getLanguageIndex() && hyperlink==t2.isHyperlink() && ((getLexeme()==null && t2.getLexeme()==null) || (getLexeme()!=null && getLexeme().equals(t2.getLexeme()))); }
@Override public Token getTokenList(Segment text, int initialTokenType, int startOffset) { Token tokens = super.getTokenList(text, initialTokenType, startOffset); if (startOffset > 0 && tokens.getType() != TokenTypes.NULL) { try { processTokens(tokens); } catch (Exception e) { LOG.error("Process tokens failed for text: {}", text, e); } } return tokens; }
/** * Returns the foreground color to use when painting a token. * * @param t The token. * @return The foreground color to use for that token. This * value is never <code>null</code>. * @see #getBackgroundForToken(Token) */ public Color getForegroundForToken(Token t) { if (getHyperlinksEnabled() && hoveredOverLinkOffset==t.getOffset() && (t.isHyperlink() || linkGeneratorResult!=null)) { return hyperlinkFG; } return getForegroundForTokenType(t.getType()); }
@Test public void testNameValuePair_happyPath() { String code = "dialog.title=Options"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertFalse(token.isHyperlink()); Assert.assertTrue(token.is(TokenTypes.RESERVED_WORD, "dialog.title")); token = token.getNextToken(); Assert.assertFalse(token.isHyperlink()); Assert.assertTrue(token.isSingleChar(TokenTypes.OPERATOR, '=')); token = token.getNextToken(); Assert.assertFalse(token.isHyperlink()); Assert.assertTrue(token.is(TokenTypes.LITERAL_STRING_DOUBLE_QUOTE, "Options")); }
@Override public Token getLastNonCommentNonWhitespaceToken() { Token last = null; for (Token t=this; t!=null && t.isPaintable(); t=t.getNextToken()) { switch (t.getType()) { case COMMENT_DOCUMENTATION: case COMMENT_EOL: case COMMENT_MULTILINE: case COMMENT_KEYWORD: case COMMENT_MARKUP: case WHITESPACE: break; default: last = t; break; } } return last; }
public static String getTextAsHtml(RSyntaxTextArea textArea, int start, int end) { // Create the selection as HTML StringBuilder sb = new StringBuilder("<pre style='") .append("font-family: \"").append(textArea.getFont().getFamily()).append("\", courier;"); if (textArea.getBackground() != null) { // May be null if it is an image sb.append(" background: ") .append(HtmlUtil.getHexString(textArea.getBackground())) .append("'>"); } Token token = textArea.getTokenListFor(start, end); for (Token t = token; t != null; t = t.getNextToken()) { if (t.isPaintable()) { if (t.isSingleChar('\n')) { sb.append("<br>"); } else { sb.append(TokenUtils.tokenToHtml(textArea, t)); } } } sb.append("</pre>"); return sb.toString(); } }
@Override public Token getLastPaintableToken() { Token t = this; while (t.isPaintable()) { Token next = t.getNextToken(); if (next==null || !next.isPaintable()) { return t; } t = next; } return null; }