public SingleArrayBlockWriter(BlockBuilder blockBuilder, int start) { super(start); this.blockBuilder = blockBuilder; this.initialBlockBuilderSize = blockBuilder.getSizeInBytes(); }
SingleMapBlockWriter(int start, BlockBuilder keyBlockBuilder, BlockBuilder valueBlockBuilder) { this.offset = start; this.keyBlockBuilder = keyBlockBuilder; this.valueBlockBuilder = valueBlockBuilder; this.initialBlockBuilderSize = keyBlockBuilder.getSizeInBytes() + valueBlockBuilder.getSizeInBytes(); }
@Override public long getSizeInBytes() { return keyBlockBuilder.getSizeInBytes() + valueBlockBuilder.getSizeInBytes() - initialBlockBuilderSize; }
@Override public long getSizeInBytes() { return blockBuilder.getSizeInBytes() - initialBlockBuilderSize; }
@Override public long getSizeInBytes() { long sizeInBytes = (Integer.BYTES + Byte.BYTES) * (long) positionCount; for (int i = 0; i < numFields; i++) { sizeInBytes += fieldBlockBuilders[i].getSizeInBytes(); } return sizeInBytes; }
@Override public long getSizeInBytes() { long currentBlockBuilderSize = 0; for (BlockBuilder fieldBlockBuilder : fieldBlockBuilders) { currentBlockBuilderSize += fieldBlockBuilder.getSizeInBytes(); } return currentBlockBuilderSize - initialBlockBuilderSize; }
SingleRowBlockWriter(int rowIndex, BlockBuilder[] fieldBlockBuilders) { super(rowIndex); this.fieldBlockBuilders = fieldBlockBuilders; long initialBlockBuilderSize = 0; for (BlockBuilder fieldBlockBuilder : fieldBlockBuilders) { initialBlockBuilderSize += fieldBlockBuilder.getSizeInBytes(); } this.initialBlockBuilderSize = initialBlockBuilderSize; }
@Override public long getSizeInBytes() { return values.getSizeInBytes() + ((Integer.BYTES + Byte.BYTES) * (long) positionCount); }
public long getSizeInBytes() { return elementBlock.getSizeInBytes(); }
@Override public long getSizeInBytes() { return keyBlockBuilder.getSizeInBytes() + valueBlockBuilder.getSizeInBytes() + (Integer.BYTES + Byte.BYTES) * (long) positionCount + Integer.BYTES * HASH_MULTIPLIER * (long) keyBlockBuilder.getPositionCount(); }
public TypedSet(Type elementType, BlockBuilder blockBuilder, int expectedSize, String functionName) { checkArgument(expectedSize >= 0, "expectedSize must not be negative"); this.elementType = requireNonNull(elementType, "elementType must not be null"); this.elementBlock = requireNonNull(blockBuilder, "blockBuilder must not be null"); this.functionName = functionName; initialElementBlockOffset = elementBlock.getPositionCount(); initialElementBlockSizeInBytes = elementBlock.getSizeInBytes(); this.size = 0; this.hashCapacity = arraySize(expectedSize, FILL_RATIO); this.maxFill = calculateMaxFill(hashCapacity); this.hashMask = hashCapacity - 1; blockPositionByHash = new IntArrayList(hashCapacity); blockPositionByHash.size(hashCapacity); for (int i = 0; i < hashCapacity; i++) { blockPositionByHash.set(i, EMPTY_SLOT); } this.containsNullElement = false; }
@Test public void testLazyBlockBuilderInitialization() { Slice[] expectedValues = createTestValue(100); BlockBuilder emptyBlockBuilder = new IntArrayBlockBuilder(null, 0); BlockBuilder blockBuilder = new IntArrayBlockBuilder(null, expectedValues.length); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); writeValues(expectedValues, blockBuilder); assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes()); assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes()); blockBuilder = blockBuilder.newBlockBuilderLike(null); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); }
@Test public void testLazyBlockBuilderInitialization() { Slice[] expectedValues = createTestValue(100); BlockBuilder emptyBlockBuilder = new VariableWidthBlockBuilder(null, 0, 0); BlockBuilder blockBuilder = new VariableWidthBlockBuilder(null, expectedValues.length, 32 * expectedValues.length); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); writeValues(expectedValues, blockBuilder); assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes()); assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes()); blockBuilder = blockBuilder.newBlockBuilderLike(null); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); }
@Test public void testLazyBlockBuilderInitialization() { Slice[] expectedValues = createTestValue(100); BlockBuilder emptyBlockBuilder = new ByteArrayBlockBuilder(null, 0); BlockBuilder blockBuilder = new ByteArrayBlockBuilder(null, expectedValues.length); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); writeValues(expectedValues, blockBuilder); assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes()); assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes()); blockBuilder = blockBuilder.newBlockBuilderLike(null); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); }
@Test public void testLazyBlockBuilderInitialization() { Slice[] expectedValues = createExpectedValues(100); BlockBuilder emptyBlockBuilder = new VariableWidthBlockBuilder(null, 0, 0); BlockBuilder blockBuilder = new VariableWidthBlockBuilder(null, expectedValues.length, 32 * expectedValues.length); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); writeValues(expectedValues, blockBuilder); assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes()); assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes()); blockBuilder = blockBuilder.newBlockBuilderLike(null); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); }
private void compactIfNecessary() { // Byte size check is needed. Otherwise, if size * 3 is small, BlockBuilder can be reallocate too often. // Position count is needed. Otherwise, for large elements, heap will be compacted every time. // Size instead of retained size is needed because default allocation size can be huge for some block builders. And the first check will become useless in such case. if (heapBlockBuilder.getSizeInBytes() < COMPACT_THRESHOLD_BYTES || heapBlockBuilder.getPositionCount() / positionCount < COMPACT_THRESHOLD_RATIO) { return; } BlockBuilder newHeapBlockBuilder = type.createBlockBuilder(null, heapBlockBuilder.getPositionCount()); for (int i = 0; i < positionCount; i++) { type.appendTo(heapBlockBuilder, heapIndex[i], newHeapBlockBuilder); heapIndex[i] = i; } heapBlockBuilder = newHeapBlockBuilder; } }
@Test public void testLazyBlockBuilderInitialization() { Slice[] expectedValues = createTestValue(100); BlockBuilder emptyBlockBuilder = new ShortArrayBlockBuilder(null, 0); BlockBuilder blockBuilder = new ShortArrayBlockBuilder(null, expectedValues.length); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); writeValues(expectedValues, blockBuilder); assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes()); assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes()); blockBuilder = blockBuilder.newBlockBuilderLike(null); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); }
private void addNewElement(int hashPosition, Block block, int position) { elementType.appendTo(block, position, elementBlock); if (elementBlock.getSizeInBytes() - initialElementBlockSizeInBytes > FOUR_MEGABYTES) { throw new PrestoException( EXCEEDED_FUNCTION_MEMORY_LIMIT, format("The input to %s is too large. More than %s of memory is needed to hold the intermediate hash set.\n", functionName, MAX_FUNCTION_MEMORY)); } blockPositionByHash.set(hashPosition, elementBlock.getPositionCount() - 1); // increase capacity, if necessary size++; if (size >= maxFill) { rehash(); } }
@Test public void testLazyBlockBuilderInitialization() { for (int fixedSize = 0; fixedSize < 20; fixedSize++) { Slice[] expectedValues = alternatingNullValues(createExpectedValues(17, fixedSize)); BlockBuilder emptyBlockBuilder = new FixedWidthBlockBuilder(fixedSize, null, 0); BlockBuilder blockBuilder = new FixedWidthBlockBuilder(fixedSize, null, expectedValues.length); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); writeValues(expectedValues, blockBuilder); assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes()); assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes()); blockBuilder = blockBuilder.newBlockBuilderLike(null); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); } }
@Test public void testLazyBlockBuilderInitialization() { long[][] expectedValues = new long[ARRAY_SIZES.length][]; Random rand = new Random(47); for (int i = 0; i < ARRAY_SIZES.length; i++) { expectedValues[i] = rand.longs(ARRAY_SIZES[i]).toArray(); } BlockBuilder emptyBlockBuilder = new ArrayBlockBuilder(BIGINT, null, 0, 0); BlockBuilder blockBuilder = new ArrayBlockBuilder(BIGINT, null, 100, 100); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); writeValues(expectedValues, blockBuilder); assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes()); assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes()); blockBuilder = blockBuilder.newBlockBuilderLike(null); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); }