@Override protected boolean isSymbolCharacter(Char ch) { return super.isSymbolCharacter(ch) && !ch.is('#'); }
protected Parser(Reader input, Scope scope) { this.scope = scope; tokenizer = new Tokenizer(input); tokenizer.setProblemCollector(errors); }
/** * Consumes the current token, expecting it to be as <tt>KEYWORD</tt> with the given content * * @param keyword the expected content of the current token */ public void consumeExpectedKeyword(String keyword) { if (current().matches(Token.TokenType.KEYWORD, keyword)) { consume(); } else { addError(current(), "Unexpected token: '%s'. Expected: '%s'", current().getSource(), keyword); } }
/** * Signals that the given token is expected. * <p> * If the current input is pointing at the specified token, it will be consumed. If not, an error will be added * to the error list and the input remains unchanged. * * @param type the type of the expected token * @param trigger the trigger of the expected token */ protected void expect(Token.TokenType type, String trigger) { if (tokenizer.current().matches(type, trigger)) { tokenizer.consume(); } else { errors.add(ParseError.error(tokenizer.current(), String.format("Unexpected token '%s'. Expected: '%s'", tokenizer.current().getSource(), trigger))); } } }
if (isAtStartOfLineComment(true)) { skipToEndOfLine(); return fetch(); if (isAtStartOfBlockComment(true)) { skipBlockComment(); return fetch(); if (isAtStartOfNumber()) { return fetchNumber(); if (isAtStartOfIdentifier()) { return fetchId(); return fetchString(); if (isAtBracket(false)) { return Token.createAndFill(Token.TokenType.SYMBOL, input.consume()); if (isAtStartOfSpecialId()) { return fetchSpecialId(); if (isSymbolCharacter(input.current())) { return fetchSymbol(); input.current().getStringValue()))); input.consume(); return fetch();
/** * Determines if the given Char is a symbol character. * <p> * By default these are all non-control characters, which don't match any other class (letter, digit, whitepsace) * * @param ch the character to check * @return <tt>true</tt> if the given character is a valid symbol character, <tt>false</tt> otherwise */ @SuppressWarnings("squid:S1067") protected boolean isSymbolCharacter(Char ch) { if (ch.isEndOfInput() || ch.isDigit() || ch.isLetter() || ch.isWhitepace()) { return false; } char c = ch.getValue(); if (Character.isISOControl(c)) { return false; } return !(isAtBracket(true) || isAtStartOfBlockComment(false) || isAtStartOfLineComment(false) || isAtStartOfNumber() || isAtStartOfIdentifier() || stringDelimiters.containsKey(ch.getValue())); }
/** * Reads and returns a special id. * * @return the parsed special id as Token */ protected Token fetchSpecialId() { Token result = Token.create(Token.TokenType.SPECIAL_ID, input.current()); result.addToTrigger(input.consume()); while (isIdentifierChar(input.current())) { result.addToContent(input.consume()); } return handleKeywords(result); }
if (tokenizer.current().isSymbol("+") && tokenizer.next().isNumber()) { tokenizer.consume(); if (tokenizer.current().isNumber()) { double value = Double.parseDouble(tokenizer.consume().getContents()); if (tokenizer.current().is(Token.TokenType.ID)) { String quantifier = tokenizer.current().getContents().intern(); if ("n" == quantifier) { value /= 1000000000d; tokenizer.consume(); } else if ("u" == quantifier) { value /= 1000000d; tokenizer.consume(); } else if ("m" == quantifier) { value /= 1000d; tokenizer.consume(); } else if ("K" == quantifier || "k" == quantifier) { value *= 1000d; tokenizer.consume(); } else if ("M" == quantifier) { value *= 1000000d; tokenizer.consume(); } else if ("G" == quantifier) { value *= 1000000000d; tokenizer.consume(); } else { Token token = tokenizer.consume(); errors.add(ParseError.error(token,
@Override protected boolean isAtBracket(boolean inSymbol) { // Treat % as single symbol so that 10%; is not tokenized to // "10", "%;" but to "10", "%", ";" // The title of this method might be a bit misleading return super.isAtBracket(inSymbol) || input.current().is('%'); }
/** * Checks if the underlying input is looking at a end of block comment * <p> * If an end of block comment is detected, any characters indicating this are consumed by this method * * @return <tt>true</tt> if the next character(s) of the input end a block comment, <tt>false</tt> otherwise */ protected boolean isAtEndOfBlockComment() { return canConsumeThisString(blockCommentEnd, true); }
@Override protected boolean isIdentifierChar(Char current) { if (super.isIdentifierChar(current)) { return true; } // CSS selectors can contain "-", "." or "#" as long as it is not the last character of the token return (current.is('-') || current.is('.') || current.is('#')) && !input.next().isWhitepace(); }
@Override protected boolean isAtStartOfIdentifier() { if (super.isAtStartOfIdentifier()) { return true; } // Support vendor specific and class selectors like -moz-border-radius or .test return (input.current().is('-') || input.current().is('.')) && input.next().isLetter(); }
@Override protected Token fetchNumber() { Token token = super.fetchNumber(); // If a number is immediately followed by % or a text like "px" - this belongs to the numeric token. if (input.current().is('%')) { token.addToContent(input.consume()); return token; } while (input.current().isLetter()) { token.addToContent(input.consume()); } return token; }
/** * Creates a new tokenizer for the given input * * @param input the input to parse. The reader will be buffered by the implementation so that it can be effectively * read character b character. */ public Tokenizer(Reader input) { this.input = new LookaheadReader(input); this.input.setProblemCollector(problemCollector); // Setup default string handling addStringDelimiter('"', '\\'); addStringDelimiter('\'', '\0'); }
if (isAtStartOfLineComment(true)) { skipToEndOfLine(); return fetch(); if (isAtStartOfBlockComment(true)) { skipBlockComment(); return fetch(); if (isAtStartOfNumber()) { return fetchNumber(); if (isAtStartOfIdentifier()) { return fetchId(); return fetchString(); if (isAtBracket(false)) { return Token.createAndFill(Token.TokenType.SYMBOL, input.consume()); if (isAtStartOfSpecialId()) { return fetchSpecialId(); if (isSymbolCharacter(input.current())) { return fetchSymbol(); input.current().getStringValue()))); input.consume(); return fetch();
/** * Determines if the given Char is a symbol character. * <p> * By default these are all non-control characters, which don't match any other class (letter, digit, whitepsace) * * @param ch the character to check * @return <tt>true</tt> if the given character is a valid symbol character, <tt>false</tt> otherwise */ @SuppressWarnings("squid:S1067") protected boolean isSymbolCharacter(Char ch) { if (ch.isEndOfInput() || ch.isDigit() || ch.isLetter() || ch.isWhitepace()) { return false; } char c = ch.getValue(); if (Character.isISOControl(c)) { return false; } return !(isAtBracket(true) || isAtStartOfBlockComment(false) || isAtStartOfLineComment(false) || isAtStartOfNumber() || isAtStartOfIdentifier() || stringDelimiters.containsKey(ch.getValue())); }
/** * Signals that the given token is expected. * <p> * If the current input is pointing at the specified token, it will be consumed. If not, an error will be added * to the error list and the input remains unchanged. * * @param type the type of the expected token * @param trigger the trigger of the expected token */ protected void expect(Token.TokenType type, String trigger) { if (tokenizer.current().matches(type, trigger)) { tokenizer.consume(); } else { errors.add(ParseError.error(tokenizer.current(), String.format("Unexpected token '%s'. Expected: '%s'", tokenizer.current().getSource(), trigger))); } } }
/** * Reads and returns a special id. * * @return the parsed special id as Token */ protected Token fetchSpecialId() { Token result = Token.create(Token.TokenType.SPECIAL_ID, input.current()); result.addToTrigger(input.consume()); while (isIdentifierChar(input.current())) { result.addToContent(input.consume()); } return handleKeywords(result); }
if (tokenizer.current().isSymbol("+") && tokenizer.next().isNumber()) { tokenizer.consume(); if (tokenizer.current().isNumber()) { double value = Double.parseDouble(tokenizer.consume().getContents()); if (tokenizer.current().is(Token.TokenType.ID)) { String quantifier = tokenizer.current().getContents().intern(); if ("n" == quantifier) { value /= 1000000000d; tokenizer.consume(); } else if ("u" == quantifier) { value /= 1000000d; tokenizer.consume(); } else if ("m" == quantifier) { value /= 1000d; tokenizer.consume(); } else if ("K" == quantifier || "k" == quantifier) { value *= 1000d; tokenizer.consume(); } else if ("M" == quantifier) { value *= 1000000d; tokenizer.consume(); } else if ("G" == quantifier) { value *= 1000000000d; tokenizer.consume(); } else { Token token = tokenizer.consume(); errors.add(ParseError.error(token,
/** * Checks if the underlying input is looking at a end of block comment * <p> * If an end of block comment is detected, any characters indicating this are consumed by this method * * @return <tt>true</tt> if the next character(s) of the input end a block comment, <tt>false</tt> otherwise */ protected boolean isAtEndOfBlockComment() { return canConsumeThisString(blockCommentEnd, true); }