public OutputStream openNext() throws IOException { output.nextFile(); return new GZIPOutputStream(output) { { this.def.setLevel(task.getLevel()); } }; }
@Override public void flush() { if (doFlush()) { buffer = allocator.allocate(); } }
@Test public void testRandomReadWrite() throws Exception { newOutputStream(); out.nextFile(); ByteArrayOutputStream expectedOut = new ByteArrayOutputStream(); Random rand = runtime.getRandom(); byte[] buffer = new byte[rand.nextInt() % 1024 + 1024]; for (int i = 0; i < 256; i++) { rand.nextBytes(buffer); expectedOut.write(buffer); out.write(buffer); } out.finish(); byte[] expected = expectedOut.toByteArray(); byte[] actual = new byte[expected.length]; newInputStream(); in.nextFile(); int pos = 0; while (pos < actual.length) { int n = in.read(actual, pos, actual.length - pos); if (n < 0) { break; } pos += n; } assertEquals(expected.length, pos); assertArrayEquals(expected, actual); }
public LineEncoder(FileOutput out, EncoderTask task) { this.newline = task.getNewline().getString(); this.underlyingFileOutput = out; this.outputStream = new FileOutputOutputStream(underlyingFileOutput, task.getBufferAllocator(), FileOutputOutputStream.CloseMode.FLUSH_FINISH); CharsetEncoder encoder = task.getCharset() .newEncoder() .onMalformedInput(CodingErrorAction.REPLACE) // TODO configurable? .onUnmappableCharacter(CodingErrorAction.REPLACE); // TODO configurable? this.writer = new BufferedWriter(new OutputStreamWriter(outputStream, encoder), 32 * 1024); }
@Override public void write(byte[] b, int off, int len) { while (true) { int available = buffer.capacity() - pos; if (available < len) { buffer.setBytes(pos, b, off, available); pos += available; len -= available; off += available; flush(); } else { buffer.setBytes(pos, b, off, len); pos += len; if (available <= len) { flush(); } break; } } }
@Override public FileOutput open(TaskSource taskSource, final FileOutput fileOutput) { final PluginTask task = taskSource.loadTask(PluginTask.class); final FileOutputOutputStream output = new FileOutputOutputStream(fileOutput, task.getBufferAllocator(), FileOutputOutputStream.CloseMode.FLUSH); return new OutputStreamFileOutput(new OutputStreamFileOutput.Provider() { public OutputStream openNext() throws IOException { output.nextFile(); return new BZip2CompressorOutputStream(output, task.getLevel()); } public void finish() throws IOException { fileOutput.finish(); } public void close() throws IOException { fileOutput.close(); } }); } }
@Override public void write(int b) { buffer.array()[buffer.offset() + pos] = (byte) b; pos++; if (pos >= buffer.capacity()) { flush(); } }
@Override public FileOutput open(TaskSource taskSource, final FileOutput fileOutput) { final PluginTask task = taskSource.loadTask(PluginTask.class); final FileOutputOutputStream output = new FileOutputOutputStream(fileOutput, task.getBufferAllocator(), FileOutputOutputStream.CloseMode.FLUSH); return new OutputStreamFileOutput(new OutputStreamFileOutput.Provider() { public OutputStream openNext() throws IOException { output.nextFile(); return new GZIPOutputStream(output) { { this.def.setLevel(task.getLevel()); } }; } public void finish() throws IOException { fileOutput.finish(); } public void close() throws IOException { fileOutput.close(); } }); } }
public void nextFile() { try { writer.flush(); } catch (IOException ex) { // unexpected throw new RuntimeException(ex); } outputStream.nextFile(); }
public void finish() { doFlush(); out.finish(); }
private void newOutputStream() { fileOutput = new MockFileOutput(); out = new FileOutputOutputStream(fileOutput, runtime.getBufferAllocator(), FileOutputOutputStream.CloseMode.CLOSE); }
public OutputStream openNext() throws IOException { output.nextFile(); return new BZip2CompressorOutputStream(output, task.getLevel()); }
@Override public void close() { switch (closeMode) { case FLUSH: doFlush(); break; case FLUSH_FINISH: doFlush(); out.finish(); break; case FLUSH_FINISH_CLOSE: doFlush(); out.finish(); out.close(); break; case CLOSE: out.close(); break; default: // Never default as all enums are listed. } buffer.release(); buffer = Buffer.EMPTY; pos = 0; } }