private Block createBlock(int currentPageSize, int fieldId) { int hiveColumnIndex = hiveColumnIndexes[fieldId]; return new LazyBlock( currentPageSize, new RcFileBlockLoader(hiveColumnIndex)); }
private static LazyBlock lazyWrapper(Block block) { return new LazyBlock(block.getPositionCount(), lazyBlock -> lazyBlock.setBlock(block)); }
private static LazyBlock lazyWrapper(Block block) { return new LazyBlock(block.getPositionCount(), lazyBlock -> lazyBlock.setBlock(block)); }
@Test public void testSelectNoneFilterLazyLoad() { PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectNoneFilter()), ImmutableList.of(new InputPageProjection(1, BIGINT))); // if channel 1 is loaded, test will fail Page inputPage = new Page(createLongSequenceBlock(0, 100), new LazyBlock(100, lazyBlock -> { throw new AssertionError("Lazy block should not be loaded"); })); LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()); Iterator<Optional<Page>> output = pageProcessor.process(SESSION, new DriverYieldSignal(), memoryContext, inputPage); assertEquals(memoryContext.getBytes(), 0); List<Optional<Page>> outputPages = ImmutableList.copyOf(output); assertEquals(outputPages.size(), 0); }
@Test public void testProjectLazyLoad() { PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectAllFilter()), ImmutableList.of(new LazyPagePageProjection()), OptionalInt.of(MAX_BATCH_SIZE)); // if channel 1 is loaded, test will fail Page inputPage = new Page(createLongSequenceBlock(0, 100), new LazyBlock(100, lazyBlock -> { throw new AssertionError("Lazy block should not be loaded"); })); LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()); Iterator<Optional<Page>> output = pageProcessor.process(SESSION, new DriverYieldSignal(), memoryContext, inputPage); List<Optional<Page>> outputPages = ImmutableList.copyOf(output); assertEquals(outputPages.size(), 1); assertPageEquals(ImmutableList.of(BIGINT), outputPages.get(0).orElse(null), new Page(createLongSequenceBlock(0, 100))); }
blocks[fieldId] = new LazyBlock(batchSize, new ParquetBlockLoader(field.get()));
Block block = dataPage.getBlock(i); if (block instanceof LazyBlock && !((LazyBlock) block).isLoaded()) { adaptedBlocks[i] = new LazyBlock(rowsToKeep.size(), new RowFilterLazyBlockLoader(dataPage.getBlock(i), rowsToKeep)); Block block = dataPage.getBlock(columnMapping.getIndex()); if (coercers[fieldId] != null) { block = new LazyBlock(batchSize, new CoercionLazyBlockLoader(block, coercers[fieldId]));
blocks[fieldId] = new LazyBlock(batchSize, new OrcBlockLoader(hiveColumnIndexes[fieldId], type));
Page input = new Page(100, inputBlock, new LazyBlock(100, lazyBlock -> { throw new AssertionError("Lazy block should not be loaded"); }));
private Page recordProcessedInput(Page page) { operatorContext.recordProcessedInput(0, page.getPositionCount()); // account processed bytes from lazy blocks only when they are loaded Block[] blocks = new Block[page.getChannelCount()]; for (int i = 0; i < page.getChannelCount(); ++i) { Block block = page.getBlock(i); if (block instanceof LazyBlock) { LazyBlock delegateLazyBlock = (LazyBlock) block; blocks[i] = new LazyBlock(page.getPositionCount(), lazyBlock -> { Block loadedBlock = delegateLazyBlock.getLoadedBlock(); operatorContext.recordProcessedInput(loadedBlock.getSizeInBytes(), 0L); lazyBlock.setBlock(loadedBlock); }); } else { operatorContext.recordProcessedInput(block.getSizeInBytes(), 0L); blocks[i] = block; } } return new Page(page.getPositionCount(), blocks); }
int fieldIndex = requestedSchema.getFieldIndex(columnNames.get(fieldId)); ColumnDescriptor columnDescriptor = requestedSchema.getColumns().get(fieldIndex); blocks[fieldId] = new LazyBlock(batchSize, new ParquetBlockLoader(columnDescriptor, type));
blocks[fieldId] = new LazyBlock(batchSize, new OrcBlockLoader(hiveColumnIndexes[fieldId], type));
blocks[fieldId] = new LazyBlock(batchSize, new OrcBlockLoader(columnIndexes[fieldId], type));