public boolean nextFile() { boolean has = inputStream.nextFile(); if (has && charset.equals(UTF_8)) { skipBom(); } return has; }
@Override public long skip(long len) { int skipped = read(null, 0, (int) Math.min(len, Integer.MAX_VALUE)); return skipped > 0 ? skipped : 0; }
@Override public void close() throws IOException { files.close(); } });
FileInputInputStream in = new FileInputInputStream(input)) { while (in.nextFile()) { final String fileName = input.hintOfCurrentInputFileNameForLogging().orElse("-");
public LineDecoder(FileInput in, DecoderTask task) { this.charset = task.getCharset(); CharsetDecoder decoder = charset .newDecoder() .onMalformedInput(CodingErrorAction.REPLACE) // TODO configurable? .onUnmappableCharacter(CodingErrorAction.REPLACE); // TODO configurable? this.inputStream = new FileInputInputStream(in); this.reader = LineReader.of( new InputStreamReader(inputStream, decoder), task.getLineDelimiterRecognized().orElse(null), 256 ); }
@Test public void testRandomReadWrite() throws Exception { newOutputStream(); out.nextFile(); ByteArrayOutputStream expectedOut = new ByteArrayOutputStream(); Random rand = runtime.getRandom(); byte[] buffer = new byte[rand.nextInt() % 1024 + 1024]; for (int i = 0; i < 256; i++) { rand.nextBytes(buffer); expectedOut.write(buffer); out.write(buffer); } out.finish(); byte[] expected = expectedOut.toByteArray(); byte[] actual = new byte[expected.length]; newInputStream(); in.nextFile(); int pos = 0; while (pos < actual.length) { int n = in.read(actual, pos, actual.length - pos); if (n < 0) { break; } pos += n; } assertEquals(expected.length, pos); assertArrayEquals(expected, actual); }
@Test public void testSkipReturnsZeroForNoData() { FileInputInputStream in = new FileInputInputStream(new MockFileInput()); assertEquals("Verify skip() returns 0 when there is no data.", 0L, in.skip(1)); }
@Override public int read() { while (pos >= buffer.limit()) { if (!nextBuffer()) { return -1; } } byte b = buffer.array()[buffer.offset() + pos]; pos++; if (pos >= buffer.limit()) { releaseBuffer(); } return b & 0xff; }
@Override public void close() { releaseBuffer(); in.close(); } }
@Override public FileInput open(TaskSource taskSource, FileInput fileInput) { PluginTask task = taskSource.loadTask(PluginTask.class); final FileInputInputStream files = new FileInputInputStream(fileInput); return new InputStreamFileInput( task.getBufferAllocator(), new InputStreamFileInput.Provider() { // Implement openNextWithHints() instead of openNext() to show file name at parser plugin loaded by FileInputPlugin // Because when using decoder, parser plugin can't get file name. @Override public InputStreamWithHints openNextWithHints() throws IOException { if (!files.nextFile()) { return null; } return new InputStreamWithHints( new BZip2CompressorInputStream(files, true), fileInput.hintOfCurrentInputFileNameForLogging().orElse(null) ); } @Override public void close() throws IOException { files.close(); } }); } }
@Override public int read(byte[] b, int off, int len) { while (pos >= buffer.limit()) { if (!nextBuffer()) { return -1; } } int remaining = buffer.limit() - pos; boolean allConsumed; if (remaining <= len) { allConsumed = true; len = remaining; } else { allConsumed = false; } if (b != null) { // b == null if skip buffer.getBytes(pos, b, off, len); } if (allConsumed) { releaseBuffer(); } else { pos += len; } return len; }
private boolean nextBuffer() { releaseBuffer(); Buffer b = in.poll(); if (b == null) { return false; } buffer = b; return true; }
@Override public FileInput open(TaskSource taskSource, FileInput fileInput) { PluginTask task = taskSource.loadTask(PluginTask.class); final FileInputInputStream files = new FileInputInputStream(fileInput); return new InputStreamFileInput( task.getBufferAllocator(), new InputStreamFileInput.Provider() { // Implement openNextWithHints() instead of openNext() to show file name at parser plugin loaded by FileInputPlugin // Because when using decoder, parser plugin can't get file name. @Override public InputStreamWithHints openNextWithHints() throws IOException { if (!files.nextFile()) { return null; } return new InputStreamWithHints( new GZIPInputStream(files, 8 * 1024), fileInput.hintOfCurrentInputFileNameForLogging().orElse(null) ); } @Override public void close() throws IOException { files.close(); } }); } }
@Override public InputStreamWithHints openNextWithHints() throws IOException { if (!files.nextFile()) { return null; } return new InputStreamWithHints( new GZIPInputStream(files, 8 * 1024), fileInput.hintOfCurrentInputFileNameForLogging().orElse(null) ); }
public boolean nextFile() { releaseBuffer(); return in.nextFile(); }
@Override public void close() throws IOException { files.close(); } });
private void newInputStream() { fileInput = new ListFileInput(fileOutput.getFiles()); in = new FileInputInputStream(fileInput); }
@Override public InputStreamWithHints openNextWithHints() throws IOException { if (!files.nextFile()) { return null; } return new InputStreamWithHints( new BZip2CompressorInputStream(files, true), fileInput.hintOfCurrentInputFileNameForLogging().orElse(null) ); }