/** * Creates a new token with the given type, using the Char a initial trigger and content. * * @param type the type if this token. The supplied Char will be used as initial part of the trigger to further * specify the token * @param ch first character of the content and trigger of this token. Also specifies the position of the token. * @return a new token which is initialized with the given Char */ public static Token createAndFill(TokenType type, Char ch) { Token result = new Token(); result.type = type; result.line = ch.getLine(); result.pos = ch.getPos(); result.contents = ch.getStringValue(); result.trigger = ch.getStringValue(); result.source = ch.toString(); return result; }
/** * Creates a new token with the given type, using the Char a initial trigger and content. * * @param type the type if this token. The supplied Char will be used as initial part of the trigger to further * specify the token * @param ch first character of the content and trigger of this token. Also specifies the position of the token. * @return a new token which is initialized with the given Char */ public static Token createAndFill(TokenType type, Char ch) { Token result = new Token(); result.type = type; result.line = ch.getLine(); result.pos = ch.getPos(); result.contents = ch.getStringValue(); result.trigger = ch.getStringValue(); result.source = ch.toString(); return result; }
/** * Reads and returns a string constant. * * @return the parsed string constant a Token */ protected Token fetchString() { char separator = input.current().getValue(); char escapeChar = stringDelimiters.get(input.current().getValue()); Token result = Token.create(Token.TokenType.STRING, input.current()); result.addToTrigger(input.consume()); while (!input.current().isNewLine() && !input.current().is(separator) && !input.current().isEndOfInput()) { if (escapeChar != '\0' && input.current().is(escapeChar)) { result.addToSource(input.consume()); if (!handleStringEscape(separator, escapeChar, result)) { problemCollector.add(ParseError.error(input.next(), String.format("Cannot use '%s' as escaped character", input.next().getStringValue()))); } } else { result.addToContent(input.consume()); } } if (input.current().is(separator)) { result.addToSource(input.consume()); } else { problemCollector.add(ParseError.error(input.current(), "Premature end of string constant")); } return result; }
/** * Reads and returns a string constant. * * @return the parsed string constant a Token */ protected Token fetchString() { char separator = input.current().getValue(); char escapeChar = stringDelimiters.get(input.current().getValue()); Token result = Token.create(Token.TokenType.STRING, input.current()); result.addToTrigger(input.consume()); while (!input.current().isNewLine() && !input.current().is(separator) && !input.current().isEndOfInput()) { if (escapeChar != '\0' && input.current().is(escapeChar)) { result.addToSource(input.consume()); if (!handleStringEscape(separator, escapeChar, result)) { problemCollector.add(ParseError.error(input.next(), String.format("Cannot use '%s' as escaped character", input.next().getStringValue()))); } } else { result.addToContent(input.consume()); } } if (input.current().is(separator)) { result.addToSource(input.consume()); } else { problemCollector.add(ParseError.error(input.current(), "Premature end of string constant")); } return result; }
/** * Reads and returns an identifier * * @return the parsed identifier as Token */ protected Token fetchId() { Token result = Token.create(Token.TokenType.ID, input.current()); result.addToContent(input.consume()); while (isIdentifierChar(input.current())) { result.addToContent(input.consume()); } if (!input.current().isEndOfInput() && specialIdTerminators.contains(input.current().getValue())) { Token specialId = Token.create(Token.TokenType.SPECIAL_ID, result); specialId.setTrigger(input.current().getStringValue()); specialId.setContent(result.getContents()); specialId.setSource(result.getContents()); specialId.addToSource(input.current()); input.consume(); return handleKeywords(specialId); } return handleKeywords(result); }
/** * Reads and returns an identifier * * @return the parsed identifier as Token */ protected Token fetchId() { Token result = Token.create(Token.TokenType.ID, input.current()); result.addToContent(input.consume()); while (isIdentifierChar(input.current())) { result.addToContent(input.consume()); } if (!input.current().isEndOfInput() && specialIdTerminators.contains(input.current().getValue())) { Token specialId = Token.create(Token.TokenType.SPECIAL_ID, result); specialId.setTrigger(input.current().getStringValue()); specialId.setContent(result.getContents()); specialId.setSource(result.getContents()); specialId.addToSource(input.current()); input.consume(); return handleKeywords(specialId); } return handleKeywords(result); }
input.current().getStringValue()))); input.consume(); return fetch();
input.current().getStringValue()))); input.consume(); return fetch();