@Override public InputStreamWithHints openNextWithHints() throws IOException { if (!files.nextFile()) { return null; } return new InputStreamWithHints( new GZIPInputStream(files, 8 * 1024), fileInput.hintOfCurrentInputFileNameForLogging().orElse(null) ); }
protected final Optional<String> getHintOfCurrentInputFileNameForLogging() { if (current != null) { return current.getHintOfCurrentInputFileNameForLogging(); } else { return Optional.empty(); } } }
public void close() { try { try { if (current != null && current.getInputStream() != null) { current.getInputStream().close(); current = null; } } finally { provider.close(); } } catch (IOException ex) { throw new RuntimeException(ex); } }
@Override public InputStreamWithHints openNextWithHints() throws IOException { if (!iterator.hasNext()) { return null; } final String key = iterator.next(); final GetObjectRequest request = new GetObjectRequest(bucket, key); S3Object object = new DefaultRetryable<S3Object>(format("Getting object '%s'", request.getKey())) { @Override public S3Object call() { return client.getObject(request); } }.executeWithCheckedException(retryExec, IOException.class); long objectSize = object.getObjectMetadata().getContentLength(); // Some plugin users are parsing this output to get file list. // Keep it for now but might be removed in the future. LOGGER.info("Open S3Object with bucket [{}], key [{}], with size [{}]", bucket, key, objectSize); InputStream inputStream = new ResumableInputStream(object.getObjectContent(), new S3InputStreamReopener(client, request, objectSize, retryExec)); return new InputStreamWithHints(inputStream, String.format("s3://%s/%s", bucket, key)); }
public Buffer poll() { if (current == null || current.getInputStream() == null) { throw new IllegalStateException("nextFile() must be called before poll()"); } Buffer buffer = allocator.allocate(); try { int n = current.getInputStream().read(buffer.array(), buffer.offset(), buffer.capacity()); if (n < 0) { return null; } buffer.limit(n); Buffer b = buffer; buffer = null; return b; } catch (IOException ex) { throw new RuntimeException(ex); } finally { if (buffer != null) { buffer.release(); buffer = null; } } }
default InputStreamWithHints openNextWithHints() throws IOException { return new InputStreamWithHints(this.openNext()); }
public boolean nextFile() { try { if (current != null && current.getInputStream() != null) { current.getInputStream().close(); current = null; } current = provider.openNextWithHints(); return current != null && current.getInputStream() != null; } catch (IOException ex) { throw new RuntimeException(ex); } }
@Override public InputStreamWithHints openNextWithHints() throws IOException { if (!files.nextFile()) { return null; } return new InputStreamWithHints( new BZip2CompressorInputStream(files, true), fileInput.hintOfCurrentInputFileNameForLogging().orElse(null) ); }