public void recordStripeWritten(FlushReason flushReason, long stripeBytes, int stripeRows, int dictionaryBytes) { getFlushStats(flushReason).recordStripeWritten(stripeBytes, stripeRows, dictionaryBytes); allFlush.recordStripeWritten(stripeBytes, stripeRows, dictionaryBytes); }
@Override public void close() throws IOException { if (closed) { return; } closed = true; stats.updateSizeInBytes(-previouslyRecordedSizeInBytes); previouslyRecordedSizeInBytes = 0; flushStripe(CLOSED); orcDataSink.close(); }
public long getRetainedBytes() { return INSTANCE_SIZE + columnWritersRetainedBytes + closedStripesRetainedBytes + orcDataSink.getRetainedSizeInBytes() + (validationBuilder == null ? 0 : validationBuilder.getRetainedSize()); }
@VisibleForTesting void readCacheAt(long offset) throws IOException { DiskRange newCacheRange = regionFinder.getRangeFor(offset); cachePosition = newCacheRange.getOffset(); cacheLength = newCacheRange.getLength(); if (cache.length < cacheLength) { cache = new byte[cacheLength]; } dataSource.readFully(newCacheRange.getOffset(), cache, 0, cacheLength); }
@Override public long getSize() { return dataSource.getSize(); }
public long getBufferedBytes() { if (direct) { return (long) (rowCount * valuesPerRow * bytesPerEntry); } int dictionaryEntries = getDictionaryEntries(); int bytesPerValue = estimateIndexBytesPerValue(dictionaryEntries); return (dictionaryEntries * bytesPerEntry) + (getNonNullValueCount() * bytesPerValue); }
@Override public OrcDataSourceId getId() { return dataSource.getId(); }
public boolean isDictionaryMemoryFull() { return optimizer.isFull(getBufferedBytes()); }
@Override public void writePage(Page page) throws IOException { writer.write(page); }
@Override public void close() throws IOException { writer.close(); } }
@Override public void close() { closed = true; buffer.close(); }
@Override public long getWrittenBytes() { return orcWriter.getWrittenBytes() + orcWriter.getBufferedBytes(); }
@Override public void readFully(long position, byte[] buffer) throws IOException { readFully(position, buffer, 0, buffer.length); }
private <K, V> void runTest(String testOrcFileName, Type keyType, Type valueType, ExpectedValuesBuilder<K, V> expectedValuesBuilder) throws Exception { List<Map<K, V>> expectedValues = expectedValuesBuilder.build(); runTest(testOrcFileName, keyType, valueType, expectedValues, false, false); runTest(testOrcFileName, keyType, valueType, expectedValues, true, false); runTest(testOrcFileName, keyType, valueType, expectedValues, false, true); }
@Override public long getRetainedBytes() { // NOTE: we do not include checkpoints because they should be small and it would be annoying to calculate the size return INSTANCE_SIZE + buffer.getRetainedSize(); }
@Override public long getBufferedBytes() { return buffer.estimateOutputDataSize() + (Long.BYTES * size); }
@Override public SliceOutput appendByte(int value) { writeByte(value); return this; }
@Override public long getRetainedBytes() { // NOTE: we do not include checkpoints because they should be small and it would be annoying to calculate the size return INSTANCE_SIZE + buffer.getRetainedSize(); }