@Override public long getBytesWritten() { return countingOut.getBytesWritten(); } }
@VisibleForTesting public OutputStream getStream() throws IOException { if (rawWriter == null) { rawWriter = new ByteCountingOutputStream(flowFileOutputStream); rawWriter.write(OrcFile.MAGIC.getBytes()); headerLength = rawWriter.getBytesWritten(); writer = new OutStream("metadata", bufferSize, codec, new DirectStream(rawWriter)); protobufWriter = CodedOutputStream.newInstance(writer); } return rawWriter; }
@Override public long writeIntermediateFooter() throws IOException { // flush any buffered rows flushStripe(); // write a footer if (stripesAtLastFlush != stripes.size()) { if (callback != null) { callback.preFooterWrite(callbackContext); } int metaLength = writeMetadata(); int footLength = writeFooter(rawWriter.getBytesWritten() - metaLength); rawWriter.write(writePostScript(footLength, metaLength)); stripesAtLastFlush = stripes.size(); rawWriter.flush(); } return rawWriter.getBytesWritten(); }
final long byteOffset = (byteCountingOut == null) ? rawOutStream.getBytesWritten() : byteCountingOut.getBytesWritten(); final TocWriter tocWriter = getTocWriter(); tocWriter.addBlockOffset(rawOutStream.getBytesWritten(), eventId); this.byteCountingOut = new ByteCountingOutputStream(writableStream, byteOffset); } else { if (tocWriter != null && eventId != null) { tocWriter.addBlockOffset(rawOutStream.getBytesWritten(), eventId);
@Override public void close() throws IOException { if (callback != null) { callback.preFooterWrite(callbackContext); } // remove us from the memory manager so that we don't get any callbacks memoryManager.removeWriter(path); // actually close the file flushStripe(); int metadataLength = writeMetadata(); int footerLength = writeFooter(rawWriter.getBytesWritten() - metadataLength); rawWriter.write(writePostScript(footerLength, metadataLength)); rawWriter.close(); }
long start = rawWriter.getBytesWritten(); long stripeLen = length; long availBlockSpace = blockSize - (start % blockSize); while (availBlockSpace > 0) { int writeLen = (int) Math.min(availBlockSpace, pad.length); rawWriter.write(pad, 0, writeLen); availBlockSpace -= writeLen; rawWriter.write(stripe); rowsInStripe = stripeStatistics.getColStats(0).getNumberOfValues(); rowCount += rowsInStripe;
public CompressableRecordWriter(final OutputStream out, final String storageLocation, final AtomicLong idGenerator, final TocWriter tocWriter, final boolean compressed, final int uncompressedBlockSize) throws IOException { super(storageLocation, tocWriter); this.fos = null; this.compressed = compressed; this.uncompressedBlockSize = uncompressedBlockSize; this.rawOutStream = new ByteCountingOutputStream(new BufferedOutputStream(out)); this.idGenerator = idGenerator; }
logger.debug("Created OutputStream using session {} for {}", new Object[] {session, this}); this.out = new ByteCountingOutputStream(rawOut); this.out.close();
@Override public synchronized void write(final byte[] b, final int off, final int len) throws IOException { super.write(b, off, len); } }
((FileOutputStream) bcos.getWrappedStream()).getFD().sync(); resourceClaimManager.freeze(scc.getResourceClaim()); bcos.close(); bcos.close(); LOG.debug("Claim lenth >= max; Closing {}", this); if (LOG.isTraceEnabled()) {
@Override public synchronized void close() throws IOException { super.close(); }
@Override public synchronized void flush() throws IOException { super.flush(); }
@Override public synchronized OutputStream getWrappedStream() { return super.getWrappedStream(); }
public static void compress(final File input, final TocReader tocReader, final File output, final TocWriter tocWriter) throws IOException { try (final InputStream fis = new FileInputStream(input); final OutputStream fos = new FileOutputStream(output); final ByteCountingOutputStream byteCountingOut = new ByteCountingOutputStream(fos)) { final long blockStartOffset = byteCountingOut.getBytesWritten();
@Override public void close() throws IOException { if (callback != null) { callback.preFooterWrite(callbackContext); } // remove us from the memory manager so that we don't get any callbacks memoryManager.removeWriter(path); // actually close the file flushStripe(); int metadataLength = writeMetadata(); int footerLength = writeFooter(rawWriter.getBytesWritten() - metadataLength); rawWriter.write(writePostScript(footerLength, metadataLength)); rawWriter.close(); }
long start = rawWriter.getBytesWritten(); final long currentStripeSize = indexSize + dataSize + footer.getSerializedSize(); final long available = blockSize - (start % blockSize); while (padding > 0) { int writeLen = (int) Math.min(padding, pad.length); rawWriter.write(pad, 0, writeLen); padding -= writeLen; protobufWriter.flush(); writer.flush(); long footerLength = rawWriter.getBytesWritten() - start - dataSize - indexSize; OrcProto.StripeInformation dirEntry = OrcProto.StripeInformation.newBuilder()
@Override public long writeIntermediateFooter() throws IOException { // flush any buffered rows flushStripe(); // write a footer if (stripesAtLastFlush != stripes.size()) { if (callback != null) { callback.preFooterWrite(callbackContext); } int metaLength = writeMetadata(); int footLength = writeFooter(rawWriter.getBytesWritten() - metaLength); rawWriter.write(writePostScript(footLength, metaLength)); stripesAtLastFlush = stripes.size(); rawWriter.flush(); } return rawWriter.getBytesWritten(); }
public CompressableRecordWriter(final File file, final AtomicLong idGenerator, final TocWriter writer, final boolean compressed, final int uncompressedBlockSize) throws IOException { super(file, writer); logger.trace("Creating Record Writer for {}", file.getName()); this.compressed = compressed; this.fos = new FileOutputStream(file); rawOutStream = new ByteCountingOutputStream(new BufferedOutputStream(fos)); this.uncompressedBlockSize = uncompressedBlockSize; this.idGenerator = idGenerator; }
logger.debug("Created OutputStream using session {} for {}", new Object[] {session, this}); this.out = new ByteCountingOutputStream(rawOut); this.out.close();