/** * Adds the given Char to the source of this token, but neither to the trigger nor to the content. * * @param ch the character to add to the source * @return <tt>this</tt> to support fluent method calls */ public Token addToSource(Char ch) { source += ch.getValue(); return this; }
/** * Adds the given Char to the source of this token, but neither to the trigger nor to the content. * * @param ch the character to add to the source * @return <tt>this</tt> to support fluent method calls */ public Token addToSource(Char ch) { source += ch.getValue(); return this; }
/** * Adds the given Char to the trigger (and the source) but not to the content * * @param ch the character to add to the trigger and source * @return <tt>this</tt> to support fluent method calls */ public Token addToTrigger(Char ch) { trigger += ch.getValue(); internTrigger = null; source += ch.getValue(); return this; }
/** * Adds the given Char to the trigger (and the source) but not to the content * * @param ch the character to add to the trigger and source * @return <tt>this</tt> to support fluent method calls */ public Token addToTrigger(Char ch) { trigger += ch.getValue(); internTrigger = null; source += ch.getValue(); return this; }
/** * Adds the given Char to the content (and the source) but not to the trigger * * @param ch the character to add to the content and source * @return <tt>this</tt> to support fluent method calls */ public Token addToContent(Char ch) { return addToContent(ch.getValue()); }
/** * Determines if the underlying input is looking at the start of a special id. * <p> * By default this is one of the given <tt>specialIdStarters</tt>. * * @return <tt>true</tt> if the current input is the start of a special id, <tt>false</tt> otherwise */ protected boolean isAtStartOfSpecialId() { return specialIdStarters.contains(input.current().getValue()); }
/** * Determines if the underlying input is looking at the start of a special id. * <p> * By default this is one of the given <tt>specialIdStarters</tt>. * * @return <tt>true</tt> if the current input is the start of a special id, <tt>false</tt> otherwise */ protected boolean isAtStartOfSpecialId() { return specialIdStarters.contains(input.current().getValue()); }
/** * Adds the given Char to the content (and the source) but not to the trigger * * @param ch the character to add to the content and source * @return <tt>this</tt> to support fluent method calls */ public Token addToContent(Char ch) { return addToContent(ch.getValue()); }
/** * Determines if the given Char is a symbol character. * <p> * By default these are all non-control characters, which don't match any other class (letter, digit, whitepsace) * * @param ch the character to check * @return <tt>true</tt> if the given character is a valid symbol character, <tt>false</tt> otherwise */ @SuppressWarnings("squid:S1067") protected boolean isSymbolCharacter(Char ch) { if (ch.isEndOfInput() || ch.isDigit() || ch.isLetter() || ch.isWhitepace()) { return false; } char c = ch.getValue(); if (Character.isISOControl(c)) { return false; } return !(isAtBracket(true) || isAtStartOfBlockComment(false) || isAtStartOfLineComment(false) || isAtStartOfNumber() || isAtStartOfIdentifier() || stringDelimiters.containsKey(ch.getValue())); }
/** * Determines if the given Char is a symbol character. * <p> * By default these are all non-control characters, which don't match any other class (letter, digit, whitepsace) * * @param ch the character to check * @return <tt>true</tt> if the given character is a valid symbol character, <tt>false</tt> otherwise */ @SuppressWarnings("squid:S1067") protected boolean isSymbolCharacter(Char ch) { if (ch.isEndOfInput() || ch.isDigit() || ch.isLetter() || ch.isWhitepace()) { return false; } char c = ch.getValue(); if (Character.isISOControl(c)) { return false; } return !(isAtBracket(true) || isAtStartOfBlockComment(false) || isAtStartOfLineComment(false) || isAtStartOfNumber() || isAtStartOfIdentifier() || stringDelimiters.containsKey(ch.getValue())); }
protected Tuple<String, String> parseFieldAndValue(LookaheadReader reader) { String field = defaultField; boolean couldBeFieldNameSoFar = true; StringBuilder valueBuilder = new StringBuilder(); while (!reader.current().isEndOfInput() && !reader.current().isWhitepace() && !reader.current().is(')')) { if (reader.current().is(':') && valueBuilder.length() > 0 && couldBeFieldNameSoFar) { field = valueBuilder.toString(); valueBuilder = new StringBuilder(); // Ignore : reader.consume(); couldBeFieldNameSoFar = false; } else { if (couldBeFieldNameSoFar) { couldBeFieldNameSoFar = reader.current().is('-', '_') || reader.current().isLetter() || reader.current().isDigit(); } valueBuilder.append(reader.consume().getValue()); } } return Tuple.create(field, valueBuilder.toString()); }
/** * Reads and returns a string constant. * * @return the parsed string constant a Token */ protected Token fetchString() { char separator = input.current().getValue(); char escapeChar = stringDelimiters.get(input.current().getValue()); Token result = Token.create(Token.TokenType.STRING, input.current()); result.addToTrigger(input.consume()); while (!input.current().isNewLine() && !input.current().is(separator) && !input.current().isEndOfInput()) { if (escapeChar != '\0' && input.current().is(escapeChar)) { result.addToSource(input.consume()); if (!handleStringEscape(separator, escapeChar, result)) { problemCollector.add(ParseError.error(input.next(), String.format("Cannot use '%s' as escaped character", input.next().getStringValue()))); } } else { result.addToContent(input.consume()); } } if (input.current().is(separator)) { result.addToSource(input.consume()); } else { problemCollector.add(ParseError.error(input.current(), "Premature end of string constant")); } return result; }
/** * Reads and returns a string constant. * * @return the parsed string constant a Token */ protected Token fetchString() { char separator = input.current().getValue(); char escapeChar = stringDelimiters.get(input.current().getValue()); Token result = Token.create(Token.TokenType.STRING, input.current()); result.addToTrigger(input.consume()); while (!input.current().isNewLine() && !input.current().is(separator) && !input.current().isEndOfInput()) { if (escapeChar != '\0' && input.current().is(escapeChar)) { result.addToSource(input.consume()); if (!handleStringEscape(separator, escapeChar, result)) { problemCollector.add(ParseError.error(input.next(), String.format("Cannot use '%s' as escaped character", input.next().getStringValue()))); } } else { result.addToContent(input.consume()); } } if (input.current().is(separator)) { result.addToSource(input.consume()); } else { problemCollector.add(ParseError.error(input.current(), "Premature end of string constant")); } return result; }
/** * Reads and returns an identifier * * @return the parsed identifier as Token */ protected Token fetchId() { Token result = Token.create(Token.TokenType.ID, input.current()); result.addToContent(input.consume()); while (isIdentifierChar(input.current())) { result.addToContent(input.consume()); } if (!input.current().isEndOfInput() && specialIdTerminators.contains(input.current().getValue())) { Token specialId = Token.create(Token.TokenType.SPECIAL_ID, result); specialId.setTrigger(input.current().getStringValue()); specialId.setContent(result.getContents()); specialId.setSource(result.getContents()); specialId.addToSource(input.current()); input.consume(); return handleKeywords(specialId); } return handleKeywords(result); }
/** * Reads and returns an identifier * * @return the parsed identifier as Token */ protected Token fetchId() { Token result = Token.create(Token.TokenType.ID, input.current()); result.addToContent(input.consume()); while (isIdentifierChar(input.current())) { result.addToContent(input.consume()); } if (!input.current().isEndOfInput() && specialIdTerminators.contains(input.current().getValue())) { Token specialId = Token.create(Token.TokenType.SPECIAL_ID, result); specialId.setTrigger(input.current().getStringValue()); specialId.setContent(result.getContents()); specialId.setSource(result.getContents()); specialId.addToSource(input.current()); input.consume(); return handleKeywords(specialId); } return handleKeywords(result); }
if (stringDelimiters.containsKey(input.current().getValue())) { return fetchString();
if (stringDelimiters.containsKey(input.current().getValue())) { return fetchString();