@Nullable private Token concatTokensUntil(Token start, String endText) { StringBuilder sb = new StringBuilder(); Token current = start; while (current != null && current.getType() != TokenTypes.NULL) { String text = current.getLexeme(); if (text != null) { sb.append(text); if (text.equals(endText)) { char[] line = sb.toString().toCharArray(); TokenImpl token = new TokenImpl(line, 0, line.length - 1, start.getOffset(), start.getType(), start.getLanguageIndex()); token.setNextToken(current.getNextToken()); return token; } } current = current.getNextToken(); } return null; } }
/** * Constructs a new <code>SyntaxView</code> wrapped around an element. * * @param elem The element representing the text to display. */ public SyntaxView(Element elem) { super(elem); tempToken = new TokenImpl(); }
/** * Adds tokens to the internal token list. This is called whenever a * request is made and no more tokens are available. */ private void augmentTokenList() { TokenImpl[] temp = new TokenImpl[size + increment]; System.arraycopy(tokenList,0, temp,0, size); size += increment; tokenList = temp; for (int i=0; i<increment; i++) { tokenList[size-i-1] = new TokenImpl(); } //System.err.println("... size up to: " + size); }
/** * Creates a new WrappedSyntaxView. Lines will be wrapped * on character boundaries. * * @param elem the element underlying the view */ public WrappedSyntaxView(Element elem) { super(elem, Y_AXIS); tempToken = new TokenImpl(); s = new Segment(); drawSeg = new Segment(); tempRect = new Rectangle(); lineCountTempToken = new TokenImpl(); }
@Test public void testGetHTMLRepresentation_problemChars() { RSyntaxTextArea textArea = new RSyntaxTextArea(); char[] ch = " &\t<>'\"/".toCharArray(); TokenImpl token = new TokenImpl(ch, 0, ch.length - 1, 0, TokenTypes.IDENTIFIER, 0); // Don't bother checking font and other styles since it may be host-specific String actual = token.getHTMLRepresentation(textArea); System.out.println(actual); Assert.assertTrue(actual.startsWith("<font")); Assert.assertTrue(actual.endsWith("> &	<>'"/</font>")); }
@Test public void testGetHTMLRepresentation_happyPath() { RSyntaxTextArea textArea = new RSyntaxTextArea(); char[] ch = "for".toCharArray(); TokenImpl token = new TokenImpl(ch, 0, 2, 0, TokenTypes.IDENTIFIER, 0); // Don't bother checking font and other styles since it may be host-specific String actual = token.getHTMLRepresentation(textArea); Assert.assertTrue(actual.startsWith("<font")); Assert.assertTrue(actual.endsWith(">for</font>")); }
@Test public void testGetTokenList_whitespaceStart_LiteralDoubleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " \"Hello world\""; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_STRING_DOUBLE_QUOTE, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_whitespaceStart_LiteralSingleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " 'Hello world'"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_CHAR, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_whitespaceStart_Variable() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " $PATH"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.VARIABLE, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_whitespaceStart_LiteralBackQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " `cat foo.txt`"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_BACKQUOTE, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_nullStart_EolComment() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "# This is a comment"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue("Unexpected token type: " + token, token.is(TokenTypes.COMMENT_EOL, text)); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_whitespaceStart_EolComment() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " # This is a comment"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue("Unexpected token type: " + token, token.is(TokenTypes.COMMENT_EOL, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_nullStart_LiteralDoubleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "\"Hello world\""; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.LITERAL_STRING_DOUBLE_QUOTE, text)); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_nullStart_LiteralSingleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "'Hello world'"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.LITERAL_CHAR, text)); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_nullStart_LiteralBackQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "`cat foo.txt`"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.LITERAL_BACKQUOTE, text)); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_identifierStart_LiteralDoubleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "foo\"Hello world\""; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.IDENTIFIER, "foo")); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_STRING_DOUBLE_QUOTE, "\"Hello world\"")); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_identifierStart_LiteralBackQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "foo`cat foo.txt`"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.IDENTIFIER, "foo")); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_BACKQUOTE, "`cat foo.txt`")); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_identifierStart_LiteralSingleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = "foo'Hello world'"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.IDENTIFIER, "foo")); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_CHAR, "'Hello world'")); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_nullStart_EscapedDoubleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); // Parsed as two identifiers: '\\' and '"' String text = "\\\""; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue("Unexpected token type: " + token, token.is(TokenTypes.IDENTIFIER, "\\")); token = token.getNextToken(); Assert.assertTrue("Unexpected token type: " + token, token.is(TokenTypes.IDENTIFIER, "\"")); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_nullStart_EscapedSingleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); // Parsed as two identifiers: '\\' and "'" String text = "\\'"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue("Unexpected token type: " + token, token.is(TokenTypes.IDENTIFIER, "\\")); token = token.getNextToken(); Assert.assertTrue("Unexpected token type: " + token, token.is(TokenTypes.IDENTIFIER, "'")); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }