@Override public boolean isLeftCurly(Token t) { return t.isSingleChar(Token.SEPARATOR, '('); }
@Override public boolean isRightCurly(Token t) { return t.isSingleChar(Token.SEPARATOR, ')'); }
/** * Returns whether a token is the left bracket token. * * @param t The token. * @return Whether the token is the left bracket token. * @see #isRightBracket(Token) */ private static boolean isLeftBracket(Token t) { return t.getType()==TokenTypes.SEPARATOR && t.isSingleChar('['); }
/** * Returns whether a token is the right bracket token. * * @param t The token. * @return Whether the token is the right bracket token. * @see #isLeftBracket(Token) */ private static boolean isRightBracket(Token t) { return t.getType()==TokenTypes.SEPARATOR && t.isSingleChar(']'); }
@Test public void testEolComments_escapedPercentNotAComment() { String code = "\\% not-comment"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.SEPARATOR, '\\')); token = token.getNextToken(); Assert.assertTrue(token.isSingleChar(TokenTypes.IDENTIFIER, '%')); }
@Test public void testNameValuePair_happyPath_withWhitespace() { String code = "dialog.title = Options"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.DATA_TYPE, "dialog.title")); token = token.getNextToken(); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.isSingleChar(TokenTypes.OPERATOR, '=')); token = token.getNextToken(); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.IDENTIFIER, "Options")); }
@Test public void testGetTokenList_whitespaceStart_LiteralDoubleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " \"Hello world\""; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_STRING_DOUBLE_QUOTE, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_whitespaceStart_LiteralSingleQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " 'Hello world'"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_CHAR, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_whitespaceStart_Variable() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " $PATH"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.VARIABLE, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_whitespaceStart_LiteralBackQuote() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " `cat foo.txt`"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.LITERAL_BACKQUOTE, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testGetTokenList_whitespaceStart_EolComment() { UnixShellTokenMaker tm = new UnixShellTokenMaker(); String text = " # This is a comment"; Segment s = createSegment(text); Token token = tm.getTokenList(s, TokenTypes.NULL, 0); Assert.assertTrue(token.isSingleChar(TokenTypes.WHITESPACE, ' ')); token = token.getNextToken(); Assert.assertTrue("Unexpected token type: " + token, token.is(TokenTypes.COMMENT_EOL, text.trim())); token = token.getNextToken(); Assert.assertEquals(new TokenImpl(), token); }
@Test public void testNameValuePair_equalSignInValue() { String code = "dialog.title=Options=hello"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.DATA_TYPE, "dialog.title")); token = token.getNextToken(); Assert.assertTrue(token.isSingleChar(TokenTypes.OPERATOR, '=')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.IDENTIFIER, "Options")); token = token.getNextToken(); Assert.assertTrue(token.isSingleChar(TokenTypes.OPERATOR, '=')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.IDENTIFIER, "hello")); }
@Test public void testNameValuePair_happyPath() { String code = "dialog.title=Options"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.DATA_TYPE, "dialog.title")); token = token.getNextToken(); Assert.assertTrue(token.isSingleChar(TokenTypes.OPERATOR, '=')); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.IDENTIFIER, "Options")); }
@Test public void testHtml_happyPath_closingTag() { String code = "</body>"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertTrue(token.is(TokenTypes.MARKUP_TAG_DELIMITER, "</")); token = token.getNextToken(); Assert.assertTrue(token.is(TokenTypes.MARKUP_TAG_NAME, "body")); token = token.getNextToken(); Assert.assertTrue(token.isSingleChar(TokenTypes.MARKUP_TAG_DELIMITER, '>')); }
@Test public void testSeparators() { String code = "[ ] { }"; Segment segment = createSegment(code); JsonTokenMaker tm = new JsonTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); String[] separators = code.split(" +"); for (int i = 0; i < separators.length; i++) { Assert.assertEquals(separators[i], token.getLexeme()); Assert.assertEquals(TokenTypes.SEPARATOR, token.getType()); // Just one extra test here Assert.assertTrue(token.isSingleChar(TokenTypes.SEPARATOR, separators[i].charAt(0))); if (i < separators.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue("Not a single space: " + token, token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } Assert.assertTrue(token.getType() == TokenTypes.NULL); }
@Test public void testSeparators() { String code = "[ ] { }"; Segment segment = createSegment(code); JshintrcTokenMaker tm = new JshintrcTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); String[] separators = code.split(" +"); for (int i = 0; i < separators.length; i++) { Assert.assertEquals(separators[i], token.getLexeme()); Assert.assertEquals(TokenTypes.SEPARATOR, token.getType()); // Just one extra test here Assert.assertTrue(token.isSingleChar(TokenTypes.SEPARATOR, separators[i].charAt(0))); if (i < separators.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue("Not a single space: " + token, token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } Assert.assertTrue(token.getType() == TokenTypes.NULL); }
@Test public void testSeparators() { String code = "[ ]"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); String[] separators = code.split(" +"); for (int i = 0; i < separators.length; i++) { Assert.assertEquals(separators[i], token.getLexeme()); Assert.assertEquals(TokenTypes.SEPARATOR, token.getType()); // Just one extra test here Assert.assertTrue(token.isSingleChar(TokenTypes.SEPARATOR, separators[i].charAt(0))); if (i < separators.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue("Not a single space: " + token, token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } Assert.assertTrue(token.getType() == TokenTypes.NULL); }
@Test public void testSeparators() { String code = "( ) [ ] { }"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); String[] separators = code.split(" +"); for (int i = 0; i < separators.length; i++) { Assert.assertEquals(separators[i], token.getLexeme()); Assert.assertEquals(TokenTypes.SEPARATOR, token.getType()); // Just one extra test here Assert.assertTrue(token.isSingleChar(TokenTypes.SEPARATOR, separators[i].charAt(0))); if (i < separators.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue("Not a single space: " + token, token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } Assert.assertTrue(token.getType() == TokenTypes.NULL); }
@Test public void testSeparators() { String code = "( ) [ ] { }"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); String[] separators = code.split(" +"); for (int i = 0; i < separators.length; i++) { Assert.assertEquals(separators[i], token.getLexeme()); Assert.assertEquals(TokenTypes.SEPARATOR, token.getType()); // Just one extra test here Assert.assertTrue(token.isSingleChar(TokenTypes.SEPARATOR, separators[i].charAt(0))); if (i < separators.length - 1) { token = token.getNextToken(); Assert.assertTrue("Not a whitespace token: " + token, token.isWhitespace()); Assert.assertTrue("Not a single space: " + token, token.is(TokenTypes.WHITESPACE, " ")); } token = token.getNextToken(); } Assert.assertTrue(token.getType() == TokenTypes.NULL); }
@Test public void testNameValuePair_happyPath() { String code = "dialog.title=Options"; Segment segment = createSegment(code); TokenMaker tm = createTokenMaker(); Token token = tm.getTokenList(segment, TokenTypes.NULL, 0); Assert.assertFalse(token.isHyperlink()); Assert.assertTrue(token.is(TokenTypes.RESERVED_WORD, "dialog.title")); token = token.getNextToken(); Assert.assertFalse(token.isHyperlink()); Assert.assertTrue(token.isSingleChar(TokenTypes.OPERATOR, '=')); token = token.getNextToken(); Assert.assertFalse(token.isHyperlink()); Assert.assertTrue(token.is(TokenTypes.LITERAL_STRING_DOUBLE_QUOTE, "Options")); }