/** * Tokenize input from a Reader as a manifest. * @param reader for input * @return token stream * @throws IOException for read errors */ public static TokenStream tokenize(Reader reader) throws IOException { StringBuilder manifestString = new StringBuilder(); char[] data = new char[8192]; int dataRead = reader.read(data); while (dataRead != -1) { manifestString.append(data, 0, dataRead); dataRead = reader.read(data); } manifestString.append("\n\n"); RecoveringManifestLexer lexer = new RecoveringManifestLexer(); lexer.initialize(); lexer.process(manifestString.toString()); return lexer.tokenStream; }
/** * Tokenize a string representing a manifest. * @param manifestString the string * @return the token stream */ public static TokenStream tokenize(String manifestString) { manifestString = manifestString + "\n\n"; RecoveringManifestLexer lexer = new RecoveringManifestLexer(); lexer.initialize(); lexer.process(manifestString); return lexer.tokenStream; }