Refine search
private Page recordProcessedInput(Page page) { operatorContext.recordProcessedInput(0, page.getPositionCount()); // account processed bytes from lazy blocks only when they are loaded Block[] blocks = new Block[page.getChannelCount()]; for (int i = 0; i < page.getChannelCount(); ++i) { Block block = page.getBlock(i); if (block instanceof LazyBlock) { LazyBlock delegateLazyBlock = (LazyBlock) block; blocks[i] = new LazyBlock(page.getPositionCount(), lazyBlock -> { Block loadedBlock = delegateLazyBlock.getLoadedBlock(); operatorContext.recordProcessedInput(loadedBlock.getSizeInBytes(), 0L); lazyBlock.setBlock(loadedBlock); }); } else { operatorContext.recordProcessedInput(block.getSizeInBytes(), 0L); blocks[i] = block; } } return new Page(page.getPositionCount(), blocks); }
private static boolean isNotLoadedLazyBlock(Block block) { return (block instanceof LazyBlock) && !((LazyBlock) block).isLoaded(); }
@Override public boolean isNull(int position) { assureLoaded(); return block.isNull(position); }
private static LazyBlock lazyWrapper(Block block) { return new LazyBlock(block.getPositionCount(), lazyBlock -> lazyBlock.setBlock(block)); }
for (int fieldId = 0; fieldId < blocks.length; fieldId++) { if (constantBlocks[fieldId] != null) { blocks[fieldId] = constantBlocks[fieldId].getRegion(0, batchSize); blocks[fieldId] = new LazyBlock(batchSize, new ParquetBlockLoader(field.get())); return new Page(batchSize, blocks); throw new PrestoException(HIVE_CURSOR_ERROR, e);
Block[] adaptedBlocks = new Block[dataPage.getChannelCount()]; for (int i = 0; i < adaptedBlocks.length; i++) { Block block = dataPage.getBlock(i); if (block instanceof LazyBlock && !((LazyBlock) block).isLoaded()) { adaptedBlocks[i] = new LazyBlock(rowsToKeep.size(), new RowFilterLazyBlockLoader(dataPage.getBlock(i), rowsToKeep)); adaptedBlocks[i] = block.getPositions(rowsToKeep.elements(), 0, rowsToKeep.size()); Block block = dataPage.getBlock(columnMapping.getIndex()); if (coercers[fieldId] != null) { block = new LazyBlock(batchSize, new CoercionLazyBlockLoader(block, coercers[fieldId])); throw new PrestoException(HIVE_CURSOR_ERROR, e);
@Override public final void load(LazyBlock lazyBlock) { if (loaded) { return; } checkState(batchId == expectedBatchId); try { Block block = parquetReader.readBlock(field); lazyBlock.setBlock(block); } catch (ParquetCorruptionException e) { throw new PrestoException(HIVE_BAD_DATA, e); } catch (IOException e) { throw new PrestoException(HIVE_CURSOR_ERROR, e); } loaded = true; } }
private Page getNonLazyPage(Page page) { ImmutableSet.Builder<Integer> builder = ImmutableSet.builder(); for (ProjectionFunction projection : projections) { builder.addAll(projection.getInputChannels()); } Set<Integer> inputChannels = builder.build(); if (inputChannels.isEmpty()) { return page; } Block[] blocks = page.getBlocks(); for (int inputChannel : inputChannels) { Block block = page.getBlock(inputChannel); if (block instanceof LazyBlock) { blocks[inputChannel] = ((LazyBlock) block).getBlock(); } } return new Page(blocks); }
@Test public void testProjectLazyLoad() { PageProcessor pageProcessor = new PageProcessor(Optional.of(new SelectAllFilter()), ImmutableList.of(new LazyPagePageProjection()), OptionalInt.of(MAX_BATCH_SIZE)); // if channel 1 is loaded, test will fail Page inputPage = new Page(createLongSequenceBlock(0, 100), new LazyBlock(100, lazyBlock -> { throw new AssertionError("Lazy block should not be loaded"); })); LocalMemoryContext memoryContext = newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()); Iterator<Optional<Page>> output = pageProcessor.process(SESSION, new DriverYieldSignal(), memoryContext, inputPage); List<Optional<Page>> outputPages = ImmutableList.copyOf(output); assertEquals(outputPages.size(), 1); assertPageEquals(ImmutableList.of(BIGINT), outputPages.get(0).orElse(null), new Page(createLongSequenceBlock(0, 100))); }
@Override public Slice getSlice(int position, int offset, int length) { assureLoaded(); return block.getSlice(position, offset, length); }
@Override public int getSliceLength(int position) { assureLoaded(); return block.getSliceLength(position); }
@Override public <T> T getObject(int position, Class<T> clazz) { assureLoaded(); return block.getObject(position, clazz); }
@Override public Block getRegion(int positionOffset, int length) { assureLoaded(); return block.getRegion(positionOffset, length); }
@Override public long getLong(int position, int offset) { assureLoaded(); return block.getLong(position, offset); }
@Override public void writeBytesTo(int position, int offset, int length, BlockBuilder blockBuilder) { assureLoaded(); block.writeBytesTo(position, offset, length, blockBuilder); }
@Override public int compareTo(int leftPosition, int leftOffset, int leftLength, Block rightBlock, int rightPosition, int rightOffset, int rightLength) { assureLoaded(); return block.compareTo(leftPosition, leftOffset, leftLength, rightBlock, rightPosition, rightOffset, rightLength); }
@Override public void load(LazyBlock lazyBlock) { if (block == null) { return; } lazyBlock.setBlock(coercer.apply(block.getLoadedBlock())); // clear reference to loader to free resources, since load was successful block = null; } }
@Override public int getInt(int position, int offset) { assureLoaded(); return block.getInt(position, offset); }
@Override public boolean equals(int position, int offset, Block otherBlock, int otherPosition, int otherOffset, int length) { assureLoaded(); return block.equals(position, offset, otherBlock, otherPosition, otherOffset, length); }
@Override public long hash(int position, int offset, int length) { assureLoaded(); return block.hash(position, offset, length); }