private static RunLengthEncodedBlock createNullRLEBlock(int positionCount) { return (RunLengthEncodedBlock) RunLengthEncodedBlock.create(BOOLEAN, null, positionCount); }
private Page createStatisticsPage(Page aggregationOutput) { int positionCount = aggregationOutput.getPositionCount(); Block[] outputBlocks = new Block[types.size()]; for (int channel = 0; channel < types.size(); channel++) { if (channel < STATS_START_CHANNEL) { outputBlocks[channel] = RunLengthEncodedBlock.create(types.get(channel), null, positionCount); } else { outputBlocks[channel] = aggregationOutput.getBlock(channel - 2); } } return new Page(positionCount, outputBlocks); }
constantBlocks[columnIndex] = RunLengthEncodedBlock.create(type, null, MAX_VECTOR_LENGTH); fieldsBuilder.add(Optional.empty());
@Test public void testAllPositionsNull() { // if there are no parameters skip this test List<Type> parameterTypes = getFunction().getParameterTypes(); if (parameterTypes.isEmpty()) { return; } Block[] blocks = new Block[parameterTypes.size()]; for (int i = 0; i < parameterTypes.size(); i++) { blocks[i] = RunLengthEncodedBlock.create(parameterTypes.get(0), null, 10); } testAggregation(getExpectedValueIncludingNulls(0, 0, 10), blocks); }
@Override public Work<GroupByIdBlock> getGroupIds(Page page) { updateGroupCount(page); return new CompletedWork<>(new GroupByIdBlock(page.getPositionCount() > 0 ? 1 : 0, RunLengthEncodedBlock.create(BIGINT, 0L, page.getPositionCount()))); }
private ColumnChunk readStruct(GroupField field) throws IOException { List<TypeSignatureParameter> fields = field.getType().getTypeSignature().getParameters(); Block[] blocks = new Block[fields.size()]; ColumnChunk columnChunk = null; List<Optional<Field>> parameters = field.getChildren(); for (int i = 0; i < fields.size(); i++) { Optional<Field> parameter = parameters.get(i); if (parameter.isPresent()) { columnChunk = readColumnChunk(parameter.get()); blocks[i] = columnChunk.getBlock(); } } for (int i = 0; i < fields.size(); i++) { if (blocks[i] == null) { blocks[i] = RunLengthEncodedBlock.create(field.getType(), null, columnChunk.getBlock().getPositionCount()); } } BooleanList structIsNull = StructColumnReader.calculateStructOffsets(field, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); boolean[] structIsNullVector = structIsNull.toBooleanArray(); Block rowBlock = RowBlock.fromFieldBlocks(structIsNullVector.length, Optional.of(structIsNullVector), blocks); return new ColumnChunk(rowBlock, columnChunk.getDefinitionLevels(), columnChunk.getRepetitionLevels()); }
blocks[fieldId] = RunLengthEncodedBlock.create(type, null, batchSize);
outputBlocks[channel] = RunLengthEncodedBlock.create(types.get(channel), null, positionCount);
@Test public void testDirectConversion() { SliceDictionaryColumnWriter writer = new SliceDictionaryColumnWriter( 0, VARCHAR, CompressionKind.NONE, toIntExact(DEFAULT_MAX_COMPRESSION_BUFFER_SIZE.toBytes()), OrcEncoding.ORC, DEFAULT_MAX_STRING_STATISTICS_LIMIT); // a single row group exceeds 2G after direct conversion byte[] value = new byte[megabytes(1)]; ThreadLocalRandom.current().nextBytes(value); Block data = RunLengthEncodedBlock.create(VARCHAR, Slices.wrappedBuffer(value), 3000); writer.beginRowGroup(); writer.writeBlock(data); writer.finishRowGroup(); assertFalse(writer.tryConvertToDirect(megabytes(64)).isPresent()); }
@Test public void testSanityRLE() { PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(field(0, BIGINT), field(1, VARCHAR)), MAX_BATCH_SIZE).get(); Slice varcharValue = Slices.utf8Slice("hello"); Page page = new Page(RunLengthEncodedBlock.create(BIGINT, 123L, 100), RunLengthEncodedBlock.create(VARCHAR, varcharValue, 100)); Page outputPage = getOnlyElement( processor.process( null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)) .orElseThrow(() -> new AssertionError("page is not present")); assertEquals(outputPage.getPositionCount(), 100); assertTrue(outputPage.getBlock(0) instanceof RunLengthEncodedBlock); assertTrue(outputPage.getBlock(1) instanceof RunLengthEncodedBlock); RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) outputPage.getBlock(0); assertEquals(BIGINT.getLong(rleBlock.getValue(), 0), 123L); RunLengthEncodedBlock rleBlock1 = (RunLengthEncodedBlock) outputPage.getBlock(1); assertEquals(VARCHAR.getSlice(rleBlock1.getValue(), 0), varcharValue); }
switch (columnMapping.getKind()) { case PREFILLED: blocks.add(RunLengthEncodedBlock.create(types[fieldId], prefilledValues[fieldId], batchSize)); break; case REGULAR:
@Test public void testSanityRLE() throws Exception { PageProcessor processor = new ExpressionCompiler(createTestMetadataManager()) .compilePageProcessor(new ConstantExpression(TRUE, BOOLEAN), ImmutableList.of(new InputReferenceExpression(0, BIGINT), new InputReferenceExpression(1, VARCHAR))).get(); Slice varcharValue = Slices.utf8Slice("hello"); Page page = new Page(RunLengthEncodedBlock.create(BIGINT, 123L, 100), RunLengthEncodedBlock.create(VARCHAR, varcharValue, 100)); Page outputPage = processor.processColumnarDictionary(null, page, ImmutableList.of(BIGINT, VARCHAR)); assertEquals(outputPage.getPositionCount(), 100); assertTrue(outputPage.getBlock(0) instanceof RunLengthEncodedBlock); assertTrue(outputPage.getBlock(1) instanceof RunLengthEncodedBlock); RunLengthEncodedBlock rleBlock = (RunLengthEncodedBlock) outputPage.getBlock(0); assertEquals(BIGINT.getLong(rleBlock.getValue(), 0), 123L); RunLengthEncodedBlock rleBlock1 = (RunLengthEncodedBlock) outputPage.getBlock(1); assertEquals(VARCHAR.getSlice(rleBlock1.getValue(), 0), varcharValue); }
@Test public void testDirectConversion() { SliceDictionaryColumnWriter writer = new SliceDictionaryColumnWriter( 0, VARCHAR, CompressionKind.NONE, toIntExact(DEFAULT_MAX_COMPRESSION_BUFFER_SIZE.toBytes()), OrcEncoding.ORC, DEFAULT_MAX_STRING_STATISTICS_LIMIT); // a single row group exceeds 2G after direct conversion byte[] value = new byte[megabytes(1)]; ThreadLocalRandom.current().nextBytes(value); Block data = RunLengthEncodedBlock.create(VARCHAR, Slices.wrappedBuffer(value), 3000); writer.beginRowGroup(); writer.writeBlock(data); writer.finishRowGroup(); assertFalse(writer.tryConvertToDirect(megabytes(64)).isPresent()); }