/** * Reads all of the lines from a {@link Readable} object. The lines do not include * line-termination characters, but do include other leading and trailing whitespace. * * <p>Does not close the {@code Readable}. If reading files or resources you should use the {@link * Files#readLines} and {@link Resources#readLines} methods. * * @param r the object to read from * @return a mutable {@link List} containing all the lines * @throws IOException if an I/O error occurs */ public static List<String> readLines(Readable r) throws IOException { List<String> result = new ArrayList<>(); LineReader lineReader = new LineReader(r); String line; while ((line = lineReader.readLine()) != null) { result.add(line); } return result; }
private MessageCheckpoint parseV2(final LineReader reader) throws IOException { final long offset = Long.parseLong(reader.readLine()); final Map<String, Long> sequences = new HashMap<>(); while (true) { final String line = reader.readLine(); if (Strings.isNullOrEmpty(line)) { break; } final List<String> parts = SLASH_SPLITTER.splitToList(line); final String subject = parts.get(0); final long maxSequence = Long.parseLong(parts.get(1)); sequences.put(subject, maxSequence); } return new MessageCheckpoint(offset, sequences); } }
//Used to determine file size and then show in progress bar, app is threaded. byte[] fileBytes = IOUtilities.streamToBytes( is ); int totalFileSize = fileBytes.length; int totalRead = 0; ByteArrayInputStream bais = new ByteArrayInputStream( fileBytes ); LineReader lr = new LineReader( bais);
private ActionCheckpoint parseV3(LineReader reader) throws IOException { final long offset = Long.parseLong(reader.readLine()); final String subjectLine = reader.readLine(); if (Strings.isNullOrEmpty(subjectLine)) { break; final int groupCount = Integer.parseInt(subjectParts.get(1)); for (int i = 0; i < groupCount; i++) { final String groupLine = reader.readLine(); final List<String> groupParts = SLASH_SPLITTER.splitToList(groupLine); final String group = groupParts.get(0); final String consumerLine = reader.readLine(); final List<String> consumerParts = SLASH_SPLITTER.splitToList(consumerLine); final String consumerId = consumerParts.get(0);
new LineReader(new StringReader(taClient.getText()))
/** * Reads all of the lines from a {@link Readable} object. The lines do not include * line-termination characters, but do include other leading and trailing whitespace. * * <p>Does not close the {@code Readable}. If reading files or resources you should use the {@link * Files#readLines} and {@link Resources#readLines} methods. * * @param r the object to read from * @return a mutable {@link List} containing all the lines * @throws IOException if an I/O error occurs */ public static List<String> readLines(Readable r) throws IOException { List<String> result = new ArrayList<>(); LineReader lineReader = new LineReader(r); String line; while ((line = lineReader.readLine()) != null) { result.add(line); } return result; }
private ActionCheckpoint parseBySplitter(final LineReader reader, final Splitter splitter) throws IOException { final Table<String, String, ConsumerGroupProgress> progresses = HashBasedTable.create(); while (true) { final String subjectLine = reader.readLine(); if (Strings.isNullOrEmpty(subjectLine)) { break; } final List<String> subjectParts = splitter.splitToList(subjectLine); final String subject = subjectParts.get(0); final int groupCount = Integer.parseInt(subjectParts.get(1)); for (int i = 0; i < groupCount; i++) { final String groupLine = reader.readLine(); final List<String> groupParts = splitter.splitToList(groupLine); final String group = groupParts.get(0); final long maxPulledMessageSequence = Long.parseLong(groupParts.get(1)); final int consumerCount = Integer.parseInt(groupParts.get(2)); final ConsumerGroupProgress progress = new ConsumerGroupProgress(subject, group, false, maxPulledMessageSequence, new HashMap<>(consumerCount)); progresses.put(subject, group, progress); final Map<String, ConsumerProgress> consumers = progress.getConsumers(); for (int j = 0; j < consumerCount; j++) { final String consumerLine = reader.readLine(); final List<String> consumerParts = splitter.splitToList(consumerLine); final String consumerId = consumerParts.get(0); final long maxAckedPullLogSequence = Long.parseLong(consumerParts.get(1)); consumers.put(consumerId, new ConsumerProgress(subject, group, consumerId, -1, maxAckedPullLogSequence)); } } } return new ActionCheckpoint(true, -1, progresses); }
StashSplitIterator(AmazonS3 s3, String bucket, String key) { InputStream rawIn = new RestartingS3InputStream(s3, bucket, key); try { // File is gzipped // Note: // Because the content may be concatenated gzip files we cannot use the default GZIPInputStream. // GzipCompressorInputStream supports concatenated gzip files. GzipCompressorInputStream gzipIn = new GzipCompressorInputStream(rawIn, true); _in = new BufferedReader(new InputStreamReader(gzipIn, Charsets.UTF_8)); // Create a line reader _reader = new LineReader(_in); } catch (Exception e) { try { Closeables.close(rawIn, true); } catch (IOException ignore) { // Won't happen, already caught and logged } throw Throwables.propagate(e); } }
/** * Reads all of the lines from a {@link Readable} object. The lines do not include * line-termination characters, but do include other leading and trailing whitespace. * * <p>Does not close the {@code Readable}. If reading files or resources you should use the {@link * Files#readLines} and {@link Resources#readLines} methods. * * @param r the object to read from * @return a mutable {@link List} containing all the lines * @throws IOException if an I/O error occurs */ public static List<String> readLines(Readable r) throws IOException { List<String> result = new ArrayList<>(); LineReader lineReader = new LineReader(r); String line; while ((line = lineReader.readLine()) != null) { result.add(line); } return result; }
StashSplitIterator(AmazonS3 s3, String bucket, String key) { InputStream rawIn = new RestartingS3InputStream(s3, bucket, key); try { // File is gzipped // Note: // Because the content may be concatenated gzip files we cannot use the default GZIPInputStream. // GzipCompressorInputStream supports concatenated gzip files. GzipCompressorInputStream gzipIn = new GzipCompressorInputStream(rawIn, true); _in = new BufferedReader(new InputStreamReader(gzipIn, Charsets.UTF_8)); // Create a line reader _reader = new LineReader(_in); } catch (Exception e) { try { Closeables.close(rawIn, true); } catch (IOException ignore) { // Won't happen, already caught and logged } throw Throwables.propagate(e); } }
/** * Streams lines from a {@link Readable} object, stopping when the processor returns {@code false} * or all lines have been read and returning the result produced by the processor. Does not close * {@code readable}. Note that this method may not fully consume the contents of {@code readable} * if the processor stops processing early. * * @throws IOException if an I/O error occurs * @since 14.0 */ @CanIgnoreReturnValue // some processors won't return a useful result public static <T> T readLines(Readable readable, LineProcessor<T> processor) throws IOException { checkNotNull(readable); checkNotNull(processor); LineReader lineReader = new LineReader(readable); String line; while ((line = lineReader.readLine()) != null) { if (!processor.processLine(line)) { break; } } return processor.getResult(); }
private Collection<Seed> readBatch() { List<Seed> seeds = Lists.newLinkedList(); try { String line; int loadSize = 0; while ((line = lineReader.readLine()) != null && loadSize < batchSize) { if (StringUtils.isBlank(line)) { continue; } loadSize++; seeds.add(new Seed(line)); } } catch (IOException ioe) { log.error("error when load init seed resource", ioe); } return seeds; }
@Override public Collection<Seed> initSeeds(VSCrawlerContext vsCrawlerContext) { Properties properties = VSCrawlerContext.vsCrawlerConfigFileWatcher.loadedProperties(); String seedFilePath = PathResolver.resolveAbsolutePath(properties.getProperty(String.format(VSCrawlerConstant.VSCRAWLER_INIT_SEED_FILE, vsCrawlerContext.getCrawlerName()))); if (StringUtils.isBlank(seedFilePath) || !new File(seedFilePath).exists()) { if (StringUtils.isNotBlank(seedFilePath)) { log.warn("can not find file:{}", seedFilePath); } seedFilePath = PathResolver.resolveAbsolutePath(filePath); } if (StringUtils.isEmpty(seedFilePath) || !new File(seedFilePath).exists()) { if (StringUtils.isNotBlank(seedFilePath)) { log.warn("can not find file:{}", seedFilePath); } return Collections.emptyList(); } vsCrawlerContext.getAutoEventRegistry().registerEvent(LoadNextBatchSeedEvent.class); Collection<Seed> seeds = null; try { fileReader = new FileReader(new File(PathResolver.resolveAbsolutePath(seedFilePath))); lineReader = new LineReader(fileReader); seeds = readBatch(); return seeds; } catch (IOException e) { log.error("error when load init seed resource"); return Collections.emptyList(); } finally { closeOrReadNextBatch(seeds, vsCrawlerContext); } }
public static Optional<String> getMasterPassword(FileSystem fs, Path masterPasswordFile) { try (Closer closer = Closer.create()) { if (!fs.exists(masterPasswordFile) || fs.getFileStatus(masterPasswordFile).isDirectory()) { LOG.warn(masterPasswordFile + " does not exist or is not a file. Cannot decrypt any encrypted password."); return Optional.absent(); } InputStream in = closer.register(fs.open(masterPasswordFile)); return Optional.of(new LineReader(new InputStreamReader(in, Charsets.UTF_8)).readLine()); } catch (IOException e) { throw new RuntimeException("Failed to obtain master password from " + masterPasswordFile, e); } }
@Override protected Map<String, Object> computeNext() { String line; try { line = _reader.readLine(); } catch (IOException e) { throw Throwables.propagate(e); } if (line == null) { try { close(); } catch (IOException ignore) { // Don't worry about this, we're done iterating anyway } return endOfData(); } //noinspection unchecked return JsonHelper.fromJson(line, Map.class); }
public GuidDatasetUrnStateStoreNameParser(FileSystem fs, Path jobStatestoreRootDir) throws IOException { this.fs = fs; this.sanitizedNameToDatasetURNMap = Maps.synchronizedBiMap(HashBiMap.<String, String>create()); this.versionIdentifier = new Path(jobStatestoreRootDir, StateStoreNameVersion.V1.getDatasetUrnNameMapFile()); if (this.fs.exists(versionIdentifier)) { this.version = StateStoreNameVersion.V1; try (InputStream in = this.fs.open(versionIdentifier)) { LineReader lineReader = new LineReader(new InputStreamReader(in, Charsets.UTF_8)); String shortenName = lineReader.readLine(); while (shortenName != null) { String datasetUrn = lineReader.readLine(); this.sanitizedNameToDatasetURNMap.put(shortenName, datasetUrn); shortenName = lineReader.readLine(); } } } else { this.version = StateStoreNameVersion.V0; } }
@Override protected Map<String, Object> computeNext() { String line; try { line = _reader.readLine(); } catch (IOException e) { throw Throwables.propagate(e); } if (line == null) { try { close(); } catch (IOException ignore) { // Don't worry about this, we're done iterating anyway } return endOfData(); } //noinspection unchecked return JsonHelper.fromJson(line, Map.class); }
private static List<String> readUsingReader(String input, int chunk, boolean asReader) throws IOException { Readable readable = asReader ? getChunkedReader(input, chunk) : getChunkedReadable(input, chunk); LineReader r = new LineReader(readable); List<String> lines = Lists.newArrayList(); String line; while ((line = r.readLine()) != null) { lines.add(line); } return lines; }
@Override public MessageCheckpoint fromBytes(final byte[] data) { try { final LineReader reader = new LineReader(new StringReader(new String(data, Charsets.UTF_8))); final int version = Integer.parseInt(reader.readLine()); switch (version) { case VERSION_V1: return new MessageCheckpoint(true, -1, new HashMap<>()); case VERSION_V2: return parseV2(reader); default: throw new RuntimeException("unknown snapshot content version " + version); } } catch (IOException e) { throw new RuntimeException(e); } }