/** * Adds the given Char to the content (and the source) but not to the trigger * * @param ch the character to add to the content and source * @return <tt>this</tt> to support fluent method calls */ public Token addToContent(Char ch) { return addToContent(ch.getValue()); }
/** * Checks if the given identifier is a keyword and returns an appropriate Token * * @param idToken the identifier to check * @return a keyword Token if the given identifier was a keyword, the original Token otherwise */ protected Token handleKeywords(Token idToken) { String keyword = keywords.get(keywordsCaseSensitive ? idToken.getContents().intern() : idToken.getContents().toLowerCase().intern()); if (keyword != null) { Token keywordToken = Token.create(Token.TokenType.KEYWORD, idToken); keywordToken.setTrigger(keyword); keywordToken.setContent(idToken.getContents()); keywordToken.setSource(idToken.getSource()); return keywordToken; } return idToken; }
/** * Determines if this token has the given type and trigger. * * @param type the expected type * @param trigger the expected trigger * @return <tt>true</tt> if this token matches the given type and trigger, <tt>false</tt> otherwise */ @SuppressWarnings("squid:S1698") public boolean matches(TokenType type, String trigger) { if (!is(type)) { return false; } if (trigger == null) { throw new IllegalArgumentException("trigger must not be null"); } return getTrigger() == trigger.intern(); }
/** * Determines if this token is a keyword. * <p> * If a list of <tt>symbols</tt> is given, this method checks that the trigger matches one of them. * * @param keywords the keywords to check for. If the list es empty, only the token type is checked. * @return <tt>true</tt> if this token is a keyword and matches one of the given <tt>keywords</tt> if the list * is not empty. */ public boolean isKeyword(String... keywords) { if (keywords.length == 0) { return is(TokenType.KEYWORD); } for (String keyword : keywords) { if (matches(TokenType.KEYWORD, keyword)) { return true; } } return false; }
/** * Reads and returns a number. * * @return the parsed number as Token */ protected Token fetchNumber() { Token result = Token.create(Token.TokenType.INTEGER, input.current()); result.addToContent(input.consume()); while (input.current().isDigit() || input.current().is(decimalSeparator) || (input.current() .is(groupingSeparator) && input.next().isDigit())) { if (input.current().is(groupingSeparator)) { result.addToSource(input.consume()); } else if (input.current().is(decimalSeparator)) { if (result.is(Token.TokenType.DECIMAL)) { problemCollector.add(ParseError.error(input.current(), "Unexpected decimal separators")); } else { Token decimalToken = Token.create(Token.TokenType.DECIMAL, result); decimalToken.setContent(result.getContents() + effectiveDecimalSeparator); decimalToken.setSource(result.getSource()); result = decimalToken; } result.addToSource(input.consume()); } else { result.addToContent(input.consume()); } } return result; }
private Expression parseAtom() { if (tokenizer.current().isNumber()) { return new Number(tokenizer.consume().getContents()); if (tokenizer.current().isSpecialIdentifier("#")) { return new Color(tokenizer.consume().getSource()); if (tokenizer.current().isIdentifier() || tokenizer.current().isString()) { return parseIdentifierOrFunctionCall(); if (tokenizer.current().isSpecialIdentifier("$")) { return new VariableReference(tokenizer.consume().getContents()); if (tokenizer.current().isSymbol("(")) { tokenizer.consumeExpectedSymbol("("); Expression expression = parseExpression(true); if (tokenizer.current().isSymbol("!") && tokenizer.next().isIdentifier()) { tokenizer.consumeExpectedSymbol("!"); return new Value("!" + tokenizer.consume().getContents()); "Unexpected token: '" + tokenizer.consume().getSource() + "'. Expected an expression."); return new Value("");
/** * Reads and returns an identifier * * @return the parsed identifier as Token */ protected Token fetchId() { Token result = Token.create(Token.TokenType.ID, input.current()); result.addToContent(input.consume()); while (isIdentifierChar(input.current())) { result.addToContent(input.consume()); } if (!input.current().isEndOfInput() && specialIdTerminators.contains(input.current().getValue())) { Token specialId = Token.create(Token.TokenType.SPECIAL_ID, result); specialId.setTrigger(input.current().getStringValue()); specialId.setContent(result.getContents()); specialId.setSource(result.getContents()); specialId.addToSource(input.current()); input.consume(); return handleKeywords(specialId); } return handleKeywords(result); }
if (tokenizer.current().isSymbol("+") && tokenizer.next().isNumber()) { if (tokenizer.current().isNumber()) { double value = Double.parseDouble(tokenizer.consume().getContents()); if (tokenizer.current().is(Token.TokenType.ID)) { String quantifier = tokenizer.current().getContents().intern(); if ("n" == quantifier) { value /= 1000000000d; errors.add(ParseError.error(token, String.format("Unexpected token: '%s'. Expected a valid quantifier.", token.getSource()))); errors.add(ParseError.error(token, String.format("Unexpected token: '%s'. Expected an expression.", token.getSource()))); return Constant.EMPTY;
private void readAttributeName(StringBuilder sb) { if (!tokenizer.current().isSymbol("]")) { if (!tokenizer.current().isIdentifier()) { tokenizer.addError(tokenizer.current(), "Unexpected token: '%s'. Expected an attribute name.", tokenizer.current().getSource()); } sb.append(tokenizer.consume().getContents()); } }
private List<String> parseSelector() { List<String> selector = new ArrayList<>(); parseSelectorPrefix(selector); while (tokenizer.more()) { if (tokenizer.current().isSymbol("{", ",")) { if (selector.isEmpty()) { tokenizer.addError(tokenizer.current(), "Unexpected end of CSS selector"); } return selector; } else if (tokenizer.current().isIdentifier() || tokenizer.current().isSpecialIdentifier("#", "@") || tokenizer.current().isNumber()) { StringBuilder sb = new StringBuilder(tokenizer.consume().getSource()); parseFilterInSelector(sb); parseOperatorInSelector(sb); selector.add(sb.toString()); } else if (tokenizer.current().isSymbol("&") || tokenizer.current().isSymbol("*")) { selector.add(tokenizer.consume().getTrigger()); } else if (tokenizer.current().isSymbol(">", "+", "~")) { selector.add(tokenizer.consume().getSource()); } else { tokenizer.addError(tokenizer.current(), "Unexpected Token: %s", tokenizer.consume().getSource()); } } return selector; }
private void parseMediaQuerySelector(Section result) { // Parse a media query like @media screen and (min-width: 1200px) tokenizer.consumeExpectedKeyword(KEYWORD_MEDIA); while (true) { if (tokenizer.current().isIdentifier()) { // Handle plain identifiers like "screen" or "print" result.addMediaQuery(new Value(tokenizer.consume().getContents())); } else if (tokenizer.current().isSymbol("(")) { parseMediaQueryFilters(result); } else { return; } // We only handle "and" as conjunction between two filters if (!tokenizer.current().isIdentifier("and")) { return; } else { tokenizer.consume(); } } }
/** * Reads and returns a string constant. * * @return the parsed string constant a Token */ protected Token fetchString() { char separator = input.current().getValue(); char escapeChar = stringDelimiters.get(input.current().getValue()); Token result = Token.create(Token.TokenType.STRING, input.current()); result.addToTrigger(input.consume()); while (!input.current().isNewLine() && !input.current().is(separator) && !input.current().isEndOfInput()) { if (escapeChar != '\0' && input.current().is(escapeChar)) { result.addToSource(input.consume()); if (!handleStringEscape(separator, escapeChar, result)) { problemCollector.add(ParseError.error(input.next(), String.format("Cannot use '%s' as escaped character", input.next().getStringValue()))); } } else { result.addToContent(input.consume()); } } if (input.current().is(separator)) { result.addToSource(input.consume()); } else { problemCollector.add(ParseError.error(input.current(), "Premature end of string constant")); } return result; }
private void parseSelectorPrefix(List<String> selector) { if (tokenizer.more() && tokenizer.current().isSymbol("[")) { StringBuilder sb = new StringBuilder(); parseFilterInSelector(sb); parseOperatorInSelector(sb); selector.add(sb.toString()); } if (tokenizer.more() && tokenizer.current().isSymbol("&")) { selector.add(tokenizer.consume().getTrigger()); } if (tokenizer.more() && tokenizer.current().isSymbol("&:")) { tokenizer.consume(); if (tokenizer.current().is(Token.TokenType.ID)) { selector.add("&"); selector.add(":" + tokenizer.consume().getContents()); } } if (tokenizer.more() && tokenizer.current().isSymbol("&::")) { tokenizer.consume(); if (tokenizer.current().is(Token.TokenType.ID)) { selector.add("&"); selector.add("::" + tokenizer.consume().getContents()); } } if (tokenizer.more() && tokenizer.current().isSymbol("::") && tokenizer.next().is(Token.TokenType.ID)) { tokenizer.consume(); selector.add("::" + tokenizer.consume().getContents()); } }
private void parseExtend(Section result) { // Parse @extend instructions like "@extend .warning" tokenizer.consumeExpectedKeyword(KEYWORD_EXTEND); if (tokenizer.current().isIdentifier() || tokenizer.current().isSpecialIdentifier("#")) { result.addExtends(tokenizer.consume().getSource()); } else { tokenizer.addError(tokenizer.current(), "Unexpected token: '" + tokenizer.current().getSource() + "'. Expected a selector to include."); } if (tokenizer.current().isSymbol(";") || !tokenizer.next().isSymbol("}")) { tokenizer.consumeExpectedSymbol(";"); } }
/** * Reads and returns a special id. * * @return the parsed special id as Token */ protected Token fetchSpecialId() { Token result = Token.create(Token.TokenType.SPECIAL_ID, input.current()); result.addToTrigger(input.consume()); while (isIdentifierChar(input.current())) { result.addToContent(input.consume()); } return handleKeywords(result); }
/** * Reads and returns a symbol. * <p> * A symbol are one or two characters, which don't match any other token type. In most cases, this will be * operators like + or *. * * @return the parsed symbol as Token */ @SuppressWarnings("squid:S1067") protected Token fetchSymbol() { Token result = Token.create(Token.TokenType.SYMBOL, input.current()); result.addToTrigger(input.consume()); if (result.isSymbol("*") && input.current().is('*') || result.isSymbol("&") && input.current().is('&') || result.isSymbol("|") && input.current().is('|') || result.isSymbol() && input.current().is('=')) { result.addToTrigger(input.consume()); } return result; }
/** * Parser rule for parsing a term. * <p> * A term is a <tt>product</tt> which might be followed by + or - as operator and another <tt>term</tt>. * * @return a term parsed from the given input */ protected Expression term() { Expression left = product(); if (tokenizer.current().isSymbol("+")) { tokenizer.consume(); Expression right = term(); return reOrder(left, right, BinaryOperation.Op.ADD); } if (tokenizer.current().isSymbol("-")) { tokenizer.consume(); Expression right = term(); return reOrder(left, right, BinaryOperation.Op.SUBTRACT); } if (tokenizer.current().isNumber()) { if (tokenizer.current().getContents().startsWith("-")) { Expression right = term(); return reOrder(left, right, BinaryOperation.Op.ADD); } } return left; }
private boolean isAtAttribute() { // an attribute has at least to start with x: y ... if (!tokenizer.current().isIdentifier() || !tokenizer.next().isSymbol(":")) { return false; } // We have to actually search for the final ";" to determine if we're // really looking at an attribute.... int i = 2; while (true) { Token next = tokenizer.next(i); if (next.isEnd() || next.isSymbol(";")) { return true; } else if (next.isSymbol("{")) { return false; } else { i++; } } }
private void consumeArgument(StringBuilder sb) { sb.append(tokenizer.consume().getSource()); int braces = 1; while (!tokenizer.current().isEnd() && braces > 0) { if (tokenizer.current().isSymbol("(")) { braces++; } if (tokenizer.current().isSymbol(")")) { braces--; } sb.append(tokenizer.consume().getSource()); } }