/** * Adds the given Char to the content (and the source) but not to the trigger * * @param ch the character to add to the content and source * @return <tt>this</tt> to support fluent method calls */ public Token addToContent(Char ch) { return addToContent(ch.getValue()); }
@Override protected boolean handleStringEscape(char separator, char escapeChar, Token stringToken) { // All escaped characters will be kept in original form... stringToken.addToContent(input.consume()); return true; }
/** * Adds the given Char to the content (and the source) but not to the trigger * * @param ch the character to add to the content and source * @return <tt>this</tt> to support fluent method calls */ public Token addToContent(Char ch) { return addToContent(ch.getValue()); }
@Override protected Token fetchNumber() { Token token = super.fetchNumber(); // If a number is immediately followed by % or a text like "px" - this belongs to the numeric token. if (input.current().is('%')) { token.addToContent(input.consume()); return token; } while (input.current().isLetter()) { token.addToContent(input.consume()); } return token; }
/** * Reads and returns a special id. * * @return the parsed special id as Token */ protected Token fetchSpecialId() { Token result = Token.create(Token.TokenType.SPECIAL_ID, input.current()); result.addToTrigger(input.consume()); while (isIdentifierChar(input.current())) { result.addToContent(input.consume()); } return handleKeywords(result); }
/** * Reads and returns a special id. * * @return the parsed special id as Token */ protected Token fetchSpecialId() { Token result = Token.create(Token.TokenType.SPECIAL_ID, input.current()); result.addToTrigger(input.consume()); while (isIdentifierChar(input.current())) { result.addToContent(input.consume()); } return handleKeywords(result); }
/** * Reads and returns an identifier * * @return the parsed identifier as Token */ protected Token fetchId() { Token result = Token.create(Token.TokenType.ID, input.current()); result.addToContent(input.consume()); while (isIdentifierChar(input.current())) { result.addToContent(input.consume()); } if (!input.current().isEndOfInput() && specialIdTerminators.contains(input.current().getValue())) { Token specialId = Token.create(Token.TokenType.SPECIAL_ID, result); specialId.setTrigger(input.current().getStringValue()); specialId.setContent(result.getContents()); specialId.setSource(result.getContents()); specialId.addToSource(input.current()); input.consume(); return handleKeywords(specialId); } return handleKeywords(result); }
/** * Reads and returns an identifier * * @return the parsed identifier as Token */ protected Token fetchId() { Token result = Token.create(Token.TokenType.ID, input.current()); result.addToContent(input.consume()); while (isIdentifierChar(input.current())) { result.addToContent(input.consume()); } if (!input.current().isEndOfInput() && specialIdTerminators.contains(input.current().getValue())) { Token specialId = Token.create(Token.TokenType.SPECIAL_ID, result); specialId.setTrigger(input.current().getStringValue()); specialId.setContent(result.getContents()); specialId.setSource(result.getContents()); specialId.addToSource(input.current()); input.consume(); return handleKeywords(specialId); } return handleKeywords(result); }
/** * Reads and returns a string constant. * * @return the parsed string constant a Token */ protected Token fetchString() { char separator = input.current().getValue(); char escapeChar = stringDelimiters.get(input.current().getValue()); Token result = Token.create(Token.TokenType.STRING, input.current()); result.addToTrigger(input.consume()); while (!input.current().isNewLine() && !input.current().is(separator) && !input.current().isEndOfInput()) { if (escapeChar != '\0' && input.current().is(escapeChar)) { result.addToSource(input.consume()); if (!handleStringEscape(separator, escapeChar, result)) { problemCollector.add(ParseError.error(input.next(), String.format("Cannot use '%s' as escaped character", input.next().getStringValue()))); } } else { result.addToContent(input.consume()); } } if (input.current().is(separator)) { result.addToSource(input.consume()); } else { problemCollector.add(ParseError.error(input.current(), "Premature end of string constant")); } return result; }
/** * Reads and returns a string constant. * * @return the parsed string constant a Token */ protected Token fetchString() { char separator = input.current().getValue(); char escapeChar = stringDelimiters.get(input.current().getValue()); Token result = Token.create(Token.TokenType.STRING, input.current()); result.addToTrigger(input.consume()); while (!input.current().isNewLine() && !input.current().is(separator) && !input.current().isEndOfInput()) { if (escapeChar != '\0' && input.current().is(escapeChar)) { result.addToSource(input.consume()); if (!handleStringEscape(separator, escapeChar, result)) { problemCollector.add(ParseError.error(input.next(), String.format("Cannot use '%s' as escaped character", input.next().getStringValue()))); } } else { result.addToContent(input.consume()); } } if (input.current().is(separator)) { result.addToSource(input.consume()); } else { problemCollector.add(ParseError.error(input.current(), "Premature end of string constant")); } return result; }
/** * Reads and returns a number. * * @return the parsed number as Token */ protected Token fetchNumber() { Token result = Token.create(Token.TokenType.INTEGER, input.current()); result.addToContent(input.consume()); while (input.current().isDigit() || input.current().is(decimalSeparator) || (input.current() .is(groupingSeparator) && input.next().isDigit())) { if (input.current().is(groupingSeparator)) { result.addToSource(input.consume()); } else if (input.current().is(decimalSeparator)) { if (result.is(Token.TokenType.DECIMAL)) { problemCollector.add(ParseError.error(input.current(), "Unexpected decimal separators")); } else { Token decimalToken = Token.create(Token.TokenType.DECIMAL, result); decimalToken.setContent(result.getContents() + effectiveDecimalSeparator); decimalToken.setSource(result.getSource()); result = decimalToken; } result.addToSource(input.consume()); } else { result.addToContent(input.consume()); } } return result; }
/** * Reads and returns a number. * * @return the parsed number as Token */ protected Token fetchNumber() { Token result = Token.create(Token.TokenType.INTEGER, input.current()); result.addToContent(input.consume()); while (input.current().isDigit() || input.current().is(decimalSeparator) || (input.current() .is(groupingSeparator) && input.next().isDigit())) { if (input.current().is(groupingSeparator)) { result.addToSource(input.consume()); } else if (input.current().is(decimalSeparator)) { if (result.is(Token.TokenType.DECIMAL)) { problemCollector.add(ParseError.error(input.current(), "Unexpected decimal separators")); } else { Token decimalToken = Token.create(Token.TokenType.DECIMAL, result); decimalToken.setContent(result.getContents() + effectiveDecimalSeparator); decimalToken.setSource(result.getSource()); result = decimalToken; } result.addToSource(input.consume()); } else { result.addToContent(input.consume()); } } return result; }
stringToken.addToContent(separator); stringToken.addToSource(input.consume()); return true;
stringToken.addToContent(separator); stringToken.addToSource(input.consume()); return true;