private static BlockBuilder createBlockBuilder() { return new VariableWidthBlockBuilder(null, 1, 1); }
private static BlockBuilder createBlockBuilder() { return new VariableWidthBlockBuilder(null, 100, 1); }
@Override public BlockBuilder createBlockBuilder(BlockBuilderStatus blockBuilderStatus, int expectedEntries, int expectedBytesPerEntry) { int maxBlockSizeInBytes; if (blockBuilderStatus == null) { maxBlockSizeInBytes = PageBuilderStatus.DEFAULT_MAX_PAGE_SIZE_IN_BYTES; } else { maxBlockSizeInBytes = blockBuilderStatus.getMaxPageSizeInBytes(); } // it is guaranteed Math.min will not overflow; safe to cast int expectedBytes = (int) min((long) expectedEntries * expectedBytesPerEntry, maxBlockSizeInBytes); return new VariableWidthBlockBuilder( blockBuilderStatus, expectedBytesPerEntry == 0 ? expectedEntries : Math.min(expectedEntries, maxBlockSizeInBytes / expectedBytesPerEntry), expectedBytes); }
private static BlockBuilder createBlockBuilderWithValues(Slice[] expectedValues) { BlockBuilder blockBuilder = new VariableWidthBlockBuilder(null, expectedValues.length, 32 * expectedValues.length); return writeValues(expectedValues, blockBuilder); }
public DictionaryBuilder(int expectedSize) { checkArgument(expectedSize >= 0, "expectedSize must not be negative"); // todo we can do better int expectedEntries = min(expectedSize, DEFAULT_MAX_PAGE_SIZE_IN_BYTES / EXPECTED_BYTES_PER_ENTRY); // it is guaranteed expectedEntries * EXPECTED_BYTES_PER_ENTRY will not overflow this.elementBlock = new VariableWidthBlockBuilder( null, expectedEntries, expectedEntries * EXPECTED_BYTES_PER_ENTRY); // first position is always null this.elementBlock.appendNull(); int hashSize = arraySize(expectedSize, FILL_RATIO); this.maxFill = calculateMaxFill(hashSize); this.hashMask = hashSize - 1; blockPositionByHash.ensureCapacity(hashSize); blockPositionByHash.fill(EMPTY_SLOT); this.containsNullElement = false; }
@Override public BlockBuilder newBlockBuilderLike(BlockBuilderStatus blockBuilderStatus) { int currentSizeInBytes = positions == 0 ? positions : (getOffset(positions) - getOffset(0)); return new VariableWidthBlockBuilder(blockBuilderStatus, calculateBlockResetSize(positions), calculateBlockResetBytes(currentSizeInBytes)); }
@Test(expectedExceptions = ArithmeticException.class) public void testOverflowOnOutput() { addToState(state, TWO.pow(126)); addToState(state, TWO.pow(126)); assertEquals(state.getOverflow(), 1); DecimalSumAggregation.outputLongDecimal(TYPE, state, new VariableWidthBlockBuilder(null, 10, 100)); }
@Test private void testGetSizeInBytes() { int numEntries = 1000; VarcharType unboundedVarcharType = createUnboundedVarcharType(); VariableWidthBlockBuilder blockBuilder = new VariableWidthBlockBuilder(null, numEntries, 20 * numEntries); for (int i = 0; i < numEntries; i++) { unboundedVarcharType.writeString(blockBuilder, String.valueOf(ThreadLocalRandom.current().nextLong())); } Block block = blockBuilder.build(); List<Block> splitQuarter = splitBlock(block, 4); long sizeInBytes = block.getSizeInBytes(); long quarter1size = splitQuarter.get(0).getSizeInBytes(); long quarter2size = splitQuarter.get(1).getSizeInBytes(); long quarter3size = splitQuarter.get(2).getSizeInBytes(); long quarter4size = splitQuarter.get(3).getSizeInBytes(); double expectedQuarterSizeMin = sizeInBytes * 0.2; double expectedQuarterSizeMax = sizeInBytes * 0.3; assertTrue(quarter1size > expectedQuarterSizeMin && quarter1size < expectedQuarterSizeMax, format("quarter1size is %s, should be between %s and %s", quarter1size, expectedQuarterSizeMin, expectedQuarterSizeMax)); assertTrue(quarter2size > expectedQuarterSizeMin && quarter2size < expectedQuarterSizeMax, format("quarter2size is %s, should be between %s and %s", quarter2size, expectedQuarterSizeMin, expectedQuarterSizeMax)); assertTrue(quarter3size > expectedQuarterSizeMin && quarter3size < expectedQuarterSizeMax, format("quarter3size is %s, should be between %s and %s", quarter3size, expectedQuarterSizeMin, expectedQuarterSizeMax)); assertTrue(quarter4size > expectedQuarterSizeMin && quarter4size < expectedQuarterSizeMax, format("quarter4size is %s, should be between %s and %s", quarter4size, expectedQuarterSizeMin, expectedQuarterSizeMax)); assertEquals(quarter1size + quarter2size + quarter3size + quarter4size, sizeInBytes); }
private Block decimalAsBlock(String value) { Slice slice = encodeScaledValue(new BigDecimal(value)); BlockBuilder blockBuilder = new VariableWidthBlockBuilder(null, 1, slice.length()); TYPE.writeSlice(blockBuilder, slice); return blockBuilder.build(); } }
private void testIsFull(PageBuilderStatus pageBuilderStatus) { BlockBuilder blockBuilder = new VariableWidthBlockBuilder(pageBuilderStatus.createBlockBuilderStatus(), 32, 1024); assertTrue(pageBuilderStatus.isEmpty()); while (!pageBuilderStatus.isFull()) { VARCHAR.writeSlice(blockBuilder, Slices.allocate(VARCHAR_VALUE_SIZE)); } assertEquals(blockBuilder.getPositionCount(), EXPECTED_ENTRY_COUNT); assertEquals(pageBuilderStatus.isFull(), true); } }
@Test public void testNewBlockBuilderLike() { int entries = 12345; double resetSkew = 1.25; BlockBuilder blockBuilder = new VariableWidthBlockBuilder(null, entries, entries); for (int i = 0; i < entries; i++) { blockBuilder.writeByte(i); blockBuilder.closeEntry(); } blockBuilder = blockBuilder.newBlockBuilderLike(null); // force to initialize capacity blockBuilder.writeByte(1); long actualArrayBytes = sizeOf(new int[(int) ceil(resetSkew * (entries + 1))]) + sizeOf(new boolean[(int) ceil(resetSkew * entries)]); long actualSliceBytes = SLICE_INSTANCE_SIZE + sizeOf(new byte[(int) ceil(resetSkew * entries)]); assertEquals(blockBuilder.getRetainedSizeInBytes(), BLOCK_BUILDER_INSTANCE_SIZE + actualSliceBytes + actualArrayBytes); }
@Test public void testLazyBlockBuilderInitialization() { Slice[] expectedValues = createTestValue(100); BlockBuilder emptyBlockBuilder = new VariableWidthBlockBuilder(null, 0, 0); BlockBuilder blockBuilder = new VariableWidthBlockBuilder(null, expectedValues.length, 32 * expectedValues.length); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); writeValues(expectedValues, blockBuilder); assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes()); assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes()); blockBuilder = blockBuilder.newBlockBuilderLike(null); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); }
private Block getSliceKeysBlockTemplate(List<DwrfSequenceEncoding> sequenceEncodings) { int bytes = 0; for (DwrfSequenceEncoding sequenceEncoding : sequenceEncodings) { bytes += sequenceEncoding.getKey().getBytesKey().size(); } VariableWidthBlockBuilder builder = new VariableWidthBlockBuilder(null, sequenceEncodings.size(), bytes); for (DwrfSequenceEncoding sequenceEncoding : sequenceEncodings) { Slice key = Slices.wrappedBuffer(sequenceEncoding.getKey().getBytesKey().toByteArray()); builder.writeBytes(key, 0, key.length()); builder.closeEntry(); } return builder.build(); }
@Test public void testLazyBlockBuilderInitialization() { Slice[] expectedValues = createExpectedValues(100); BlockBuilder emptyBlockBuilder = new VariableWidthBlockBuilder(null, 0, 0); BlockBuilder blockBuilder = new VariableWidthBlockBuilder(null, expectedValues.length, 32 * expectedValues.length); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); writeValues(expectedValues, blockBuilder); assertTrue(blockBuilder.getSizeInBytes() > emptyBlockBuilder.getSizeInBytes()); assertTrue(blockBuilder.getRetainedSizeInBytes() > emptyBlockBuilder.getRetainedSizeInBytes()); blockBuilder = blockBuilder.newBlockBuilderLike(null); assertEquals(blockBuilder.getSizeInBytes(), emptyBlockBuilder.getSizeInBytes()); assertEquals(blockBuilder.getRetainedSizeInBytes(), emptyBlockBuilder.getRetainedSizeInBytes()); }
private static Block createSingleValueBlock(Slice expectedValue) { BlockBuilder blockBuilder = new VariableWidthBlockBuilder(null, 1, expectedValue.length()); blockBuilder.writeBytes(expectedValue, 0, expectedValue.length()).closeEntry(); return blockBuilder.build(); }
@Override public BlockBuilder createBlockBuilder(BlockBuilderStatus blockBuilderStatus, int expectedEntries, int expectedBytesPerEntry) { int maxBlockSizeInBytes; if (blockBuilderStatus == null) { maxBlockSizeInBytes = PageBuilderStatus.DEFAULT_MAX_PAGE_SIZE_IN_BYTES; } else { maxBlockSizeInBytes = blockBuilderStatus.getMaxPageSizeInBytes(); } // it is guaranteed Math.min will not overflow; safe to cast int expectedBytes = (int) min((long) expectedEntries * expectedBytesPerEntry, maxBlockSizeInBytes); return new VariableWidthBlockBuilder( blockBuilderStatus, expectedBytesPerEntry == 0 ? expectedEntries : Math.min(expectedEntries, maxBlockSizeInBytes / expectedBytesPerEntry), expectedBytes); }
@Override public BlockBuilder newBlockBuilderLike(BlockBuilderStatus blockBuilderStatus) { int currentSizeInBytes = positions == 0 ? positions : (getOffset(positions) - getOffset(0)); return new VariableWidthBlockBuilder(blockBuilderStatus, calculateBlockResetSize(positions), calculateBlockResetBytes(currentSizeInBytes)); }
private Block getSliceKeysBlockTemplate(List<DwrfSequenceEncoding> sequenceEncodings) { int bytes = 0; for (DwrfSequenceEncoding sequenceEncoding : sequenceEncodings) { bytes += sequenceEncoding.getKey().getBytesKey().size(); } VariableWidthBlockBuilder builder = new VariableWidthBlockBuilder(null, sequenceEncodings.size(), bytes); for (DwrfSequenceEncoding sequenceEncoding : sequenceEncodings) { Slice key = Slices.wrappedBuffer(sequenceEncoding.getKey().getBytesKey().toByteArray()); builder.writeBytes(key, 0, key.length()); builder.closeEntry(); } return builder.build(); }
private static Block createSingleValueBlock(Slice expectedValue) { BlockBuilder blockBuilder = new VariableWidthBlockBuilder(new BlockBuilderStatus(), 1, expectedValue.length()); blockBuilder.writeBytes(expectedValue, 0, expectedValue.length()).closeEntry(); return blockBuilder.build(); } }
private static BlockBuilder createBlockBuilderWithValues(Slice[] expectedValues) { VariableWidthBlockBuilder blockBuilder = new VariableWidthBlockBuilder(new BlockBuilderStatus(), expectedValues.length, 32); for (Slice expectedValue : expectedValues) { if (expectedValue == null) { blockBuilder.appendNull(); } else { blockBuilder.writeBytes(expectedValue, 0, expectedValue.length()).closeEntry(); } } return blockBuilder; } }