SingleMapBlockWriter(int start, BlockBuilder keyBlockBuilder, BlockBuilder valueBlockBuilder) { this.offset = start; this.keyBlockBuilder = keyBlockBuilder; this.valueBlockBuilder = valueBlockBuilder; this.initialBlockBuilderSize = keyBlockBuilder.getSizeInBytes() + valueBlockBuilder.getSizeInBytes(); }
@Override public long getSizeInBytes() { return values.getSizeInBytes() + ((Integer.BYTES + Byte.BYTES) * (long) positionCount); }
SingleRowBlockWriter(int rowIndex, BlockBuilder[] fieldBlockBuilders) { super(rowIndex); this.fieldBlockBuilders = fieldBlockBuilders; long initialBlockBuilderSize = 0; for (BlockBuilder fieldBlockBuilder : fieldBlockBuilders) { initialBlockBuilderSize += fieldBlockBuilder.getSizeInBytes(); } this.initialBlockBuilderSize = initialBlockBuilderSize; }
@Override public long getSizeInBytes() { return keyBlockBuilder.getSizeInBytes() + valueBlockBuilder.getSizeInBytes() - initialBlockBuilderSize; }
public SingleArrayBlockWriter(BlockBuilder blockBuilder, int start) { super(start); this.blockBuilder = blockBuilder; this.initialBlockBuilderSize = blockBuilder.getSizeInBytes(); }
@Override public long getSizeInBytes() { return blockBuilder.getSizeInBytes() - initialBlockBuilderSize; }
@Override public long getSizeInBytes() { long sizeInBytes = (Integer.BYTES + Byte.BYTES) * (long) positionCount; for (int i = 0; i < numFields; i++) { sizeInBytes += fieldBlockBuilders[i].getSizeInBytes(); } return sizeInBytes; }
@Override public long getSizeInBytes() { long currentBlockBuilderSize = 0; for (BlockBuilder fieldBlockBuilder : fieldBlockBuilders) { currentBlockBuilderSize += fieldBlockBuilder.getSizeInBytes(); } return currentBlockBuilderSize - initialBlockBuilderSize; }
public long getSizeInBytes() { return elementBlock.getSizeInBytes(); }
public long getSizeInBytes() { return elementBlock.getSizeInBytes(); }
@Override public long getSizeInBytes() { return keyBlockBuilder.getSizeInBytes() + valueBlockBuilder.getSizeInBytes() + (Integer.BYTES + Byte.BYTES) * (long) positionCount + Integer.BYTES * HASH_MULTIPLIER * (long) keyBlockBuilder.getPositionCount(); }
@Test public void testLazyBlockBuilderInitialization() { Slice[] expectedValues = createTestValue(100); BlockBuilder emptyBlockBuilder = new VariableWidthBlockBuilder(null, 0, 0); BlockBuilder blockBuilder = new VariableWidthBlockBuilder(null, expectedValues.length, 32 * expectedValues.length); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); writeValues(expectedValues, blockBuilder); assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes()); assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes()); blockBuilder = blockBuilder.newBlockBuilderLike(null); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); }
@Test public void testLazyBlockBuilderInitialization() { Slice[] expectedValues = createExpectedValues(100); BlockBuilder emptyBlockBuilder = new VariableWidthBlockBuilder(null, 0, 0); BlockBuilder blockBuilder = new VariableWidthBlockBuilder(null, expectedValues.length, 32 * expectedValues.length); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); writeValues(expectedValues, blockBuilder); assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes()); assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes()); blockBuilder = blockBuilder.newBlockBuilderLike(null); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); }
@Test public void testLazyBlockBuilderInitialization() { Slice[] expectedValues = createTestValue(100); BlockBuilder emptyBlockBuilder = new IntArrayBlockBuilder(null, 0); BlockBuilder blockBuilder = new IntArrayBlockBuilder(null, expectedValues.length); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); writeValues(expectedValues, blockBuilder); assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes()); assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes()); blockBuilder = blockBuilder.newBlockBuilderLike(null); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); }
@Test public void testLazyBlockBuilderInitialization() { Slice[] expectedValues = createTestValue(100); BlockBuilder emptyBlockBuilder = new ByteArrayBlockBuilder(null, 0); BlockBuilder blockBuilder = new ByteArrayBlockBuilder(null, expectedValues.length); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); writeValues(expectedValues, blockBuilder); assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes()); assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes()); blockBuilder = blockBuilder.newBlockBuilderLike(null); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); }
@Test public void testLazyBlockBuilderInitialization() { Slice[] expectedValues = createTestValue(100); BlockBuilder emptyBlockBuilder = new IntArrayBlockBuilder(null, 0); BlockBuilder blockBuilder = new IntArrayBlockBuilder(null, expectedValues.length); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); writeValues(expectedValues, blockBuilder); assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes()); assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes()); blockBuilder = blockBuilder.newBlockBuilderLike(null); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); }
@Test public void testLazyBlockBuilderInitialization() { Slice[] expectedValues = createExpectedValues(100); BlockBuilder emptyBlockBuilder = new VariableWidthBlockBuilder(null, 0, 0); BlockBuilder blockBuilder = new VariableWidthBlockBuilder(null, expectedValues.length, 32 * expectedValues.length); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); writeValues(expectedValues, blockBuilder); assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes()); assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes()); blockBuilder = blockBuilder.newBlockBuilderLike(null); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); }
private void compactIfNecessary() { // Byte size check is needed. Otherwise, if size * 3 is small, BlockBuilder can be reallocate too often. // Position count is needed. Otherwise, for large elements, heap will be compacted every time. // Size instead of retained size is needed because default allocation size can be huge for some block builders. And the first check will become useless in such case. if (heapBlockBuilder.getSizeInBytes() < COMPACT_THRESHOLD_BYTES || heapBlockBuilder.getPositionCount() / positionCount < COMPACT_THRESHOLD_RATIO) { return; } BlockBuilder newHeapBlockBuilder = type.createBlockBuilder(null, heapBlockBuilder.getPositionCount()); for (int i = 0; i < positionCount; i++) { type.appendTo(heapBlockBuilder, heapIndex[i], newHeapBlockBuilder); heapIndex[i] = i; } heapBlockBuilder = newHeapBlockBuilder; } }
@Test public void testLazyBlockBuilderInitialization() { Slice[] expectedValues = createTestValue(100); BlockBuilder emptyBlockBuilder = new ShortArrayBlockBuilder(null, 0); BlockBuilder blockBuilder = new ShortArrayBlockBuilder(null, expectedValues.length); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); writeValues(expectedValues, blockBuilder); assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes()); assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes()); blockBuilder = blockBuilder.newBlockBuilderLike(null); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); }
@Test public void testLazyBlockBuilderInitialization() { Slice[] expectedValues = createTestValue(100); BlockBuilder emptyBlockBuilder = new ByteArrayBlockBuilder(null, 0); BlockBuilder blockBuilder = new ByteArrayBlockBuilder(null, expectedValues.length); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); writeValues(expectedValues, blockBuilder); assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes()); assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes()); blockBuilder = blockBuilder.newBlockBuilderLike(null); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); }