private static void skipPages(ConnectorPageSource pageSource, int skipPages) { for (int i = 0; i < skipPages; i++) { checkState(!pageSource.isFinished(), "pageSource is unexpectedly finished"); pageSource.getNextPage(); } }
@Override public void close() throws IOException { delegate.close(); } }
@Override public Page getOutput() { Page page = pageSource.getNextPage(); if (page == null) { return null; } // update operator stats long endCompletedBytes = pageSource.getCompletedBytes(); long endReadTimeNanos = pageSource.getReadTimeNanos(); operatorContext.recordRawInputWithTiming(endCompletedBytes - completedBytes, endReadTimeNanos - readTimeNanos); operatorContext.recordProcessedInput(page.getSizeInBytes(), page.getPositionCount()); completedBytes = endCompletedBytes; readTimeNanos = endReadTimeNanos; // assure the page is in memory before handing to another operator page = page.getLoadedPage(); return page; }
@Override public boolean isFinished() { if (!finished) { finished = (source != null) && source.isFinished(); if (source != null) { systemMemoryContext.setBytes(source.getSystemMemoryUsage()); } } return finished; }
@Override public void finish() { finished = true; blocked.set(null); if (source != null) { try { source.close(); } catch (IOException e) { throw new UncheckedIOException(e); } systemMemoryContext.setBytes(source.getSystemMemoryUsage()); } }
@Override public Page getOutput() { if (split == null) { return null; } if (source == null) { source = pageSourceProvider.createPageSource(operatorContext.getSession(), split, columns); } Page page = source.getNextPage(); if (page != null) { // assure the page is in memory before handing to another operator page = page.getLoadedPage(); // update operator stats long endCompletedBytes = source.getCompletedBytes(); long endReadTimeNanos = source.getReadTimeNanos(); operatorContext.recordRawInputWithTiming(endCompletedBytes - completedBytes, endReadTimeNanos - readTimeNanos); operatorContext.recordProcessedInput(page.getSizeInBytes(), page.getPositionCount()); completedBytes = endCompletedBytes; readTimeNanos = endReadTimeNanos; } // updating system memory usage should happen after page is loaded. systemMemoryContext.setBytes(source.getSystemMemoryUsage()); return page; } }
assertEquals(pageSource.getSystemMemoryUsage(), 0); assertFalse(pageSource.isFinished()); Page page = pageSource.getNextPage(); assertNotNull(page); Block block = page.getBlock(1); assertBetweenInclusive(pageSource.getSystemMemoryUsage(), 180000L, 189999L); // Memory usage before lazy-loading the block createUnboundedVarcharType().getSlice(block, block.getPositionCount() - 1); // trigger loading for lazy block memoryUsage = pageSource.getSystemMemoryUsage(); assertEquals(pageSource.getSystemMemoryUsage(), memoryUsage); createUnboundedVarcharType().getSlice(block, block.getPositionCount() - 1); // trigger loading for lazy block assertEquals(pageSource.getSystemMemoryUsage(), memoryUsage); assertFalse(pageSource.isFinished()); Page page = pageSource.getNextPage(); assertNotNull(page); Block block = page.getBlock(1); assertBetweenInclusive(pageSource.getSystemMemoryUsage(), 180000L, 189999L); // Memory usage before lazy-loading the block createUnboundedVarcharType().getSlice(block, block.getPositionCount() - 1); // trigger loading for lazy block memoryUsage = pageSource.getSystemMemoryUsage(); assertEquals(pageSource.getSystemMemoryUsage(), memoryUsage); createUnboundedVarcharType().getSlice(block, block.getPositionCount() - 1); // trigger loading for lazy block assertEquals(pageSource.getSystemMemoryUsage(), memoryUsage); assertFalse(pageSource.isFinished());
try (ConnectorPageSource pageSource = pageSourceManager.createPageSource(session, split, ImmutableList.of(kdbTreeColumn))) { do { getFutureValue(pageSource.isBlocked()); Page page = pageSource.getNextPage(); if (page != null && page.getPositionCount() > 0) { checkSpatialPartitioningTable(!kdbTree.isPresent(), "Expected exactly one row for table %s, but found more", name); while (!pageSource.isFinished());
static class HiveTransaction implements Transaction { private final HiveTransactionManager transactionManager; private final ConnectorTransactionHandle transactionHandle; private boolean closed; public HiveTransaction(HiveTransactionManager transactionManager, HiveMetadata hiveMetadata) { this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); this.transactionHandle = new HiveTransactionHandle(); transactionManager.put(transactionHandle, hiveMetadata); getMetastore().testOnlyThrowOnCleanupFailures(); } @Override public ConnectorMetadata getMetadata() { return transactionManager.get(transactionHandle); } @Override public SemiTransactionalHiveMetastore getMetastore(String schema) { return getMetastore(); } private SemiTransactionalHiveMetastore getMetastore() { return ((HiveMetadata) transactionManager.get(transactionHandle)).getMetastore();
@Override public boolean isFinished() { return pageSource.isFinished(); }
@Override public Page getNextPage() { Page nextPage = delegate.getNextPage(); if (nextPage == null) { return null; } Block[] blocks = Arrays.stream(delegateFieldIndex) .mapToObj(nextPage::getBlock) .toArray(Block[]::new); return new Page(nextPage.getPositionCount(), blocks); }
@Override public long getCompletedBytes() { return delegate.getCompletedBytes(); }
@Override public long getSystemMemoryUsage() { return delegate.getSystemMemoryUsage(); }
@Override public long getReadTimeNanos() { return delegate.getReadTimeNanos(); }
private Page processPageSource() { DriverYieldSignal yieldSignal = operatorContext.getDriverContext().getYieldSignal(); if (!finishing && mergingOutput.needsInput() && !yieldSignal.isSet()) { Page page = pageSource.getNextPage(); finishing = pageSource.isFinished(); pageSourceMemoryContext.setBytes(pageSource.getSystemMemoryUsage()); if (page != null) { page = recordProcessedInput(page); // update operator stats long endCompletedBytes = pageSource.getCompletedBytes(); long endReadTimeNanos = pageSource.getReadTimeNanos(); operatorContext.recordRawInputWithTiming(endCompletedBytes - completedBytes, endReadTimeNanos - readTimeNanos); completedBytes = endCompletedBytes; readTimeNanos = endReadTimeNanos; Iterator<Optional<Page>> output = pageProcessor.process(operatorContext.getSession().toConnectorSession(), yieldSignal, pageProcessorMemoryContext, page); mergingOutput.addInput(output); } if (finishing) { mergingOutput.finish(); } } Page result = mergingOutput.getOutput(); outputMemoryContext.setBytes(mergingOutput.getRetainedSizeInBytes() + pageProcessorMemoryContext.getBytes()); return result; }
assertEquals(pageSource.getSystemMemoryUsage(), 0); assertFalse(pageSource.isFinished()); Page page = pageSource.getNextPage(); assertNotNull(page); Block block = page.getBlock(1); if (memoryUsage == -1) { assertBetweenInclusive(pageSource.getSystemMemoryUsage(), 180000L, 189999L); // Memory usage before lazy-loading the block VARCHAR.getSlice(block, block.getPositionCount() - 1); // trigger loading for lazy block memoryUsage = pageSource.getSystemMemoryUsage(); assertEquals(pageSource.getSystemMemoryUsage(), memoryUsage); VARCHAR.getSlice(block, block.getPositionCount() - 1); // trigger loading for lazy block assertEquals(pageSource.getSystemMemoryUsage(), memoryUsage); assertFalse(pageSource.isFinished()); Page page = pageSource.getNextPage(); assertNotNull(page); Block block = page.getBlock(1); if (memoryUsage == -1) { assertBetweenInclusive(pageSource.getSystemMemoryUsage(), 180000L, 189999L); // Memory usage before lazy-loading the block VARCHAR.getSlice(block, block.getPositionCount() - 1); // trigger loading for lazy block memoryUsage = pageSource.getSystemMemoryUsage(); assertEquals(pageSource.getSystemMemoryUsage(), memoryUsage); VARCHAR.getSlice(block, block.getPositionCount() - 1); // trigger loading for lazy block assertEquals(pageSource.getSystemMemoryUsage(), memoryUsage); assertFalse(pageSource.isFinished());
static class HiveTransaction implements Transaction { private final HiveTransactionManager transactionManager; private final ConnectorTransactionHandle transactionHandle; private boolean closed; public HiveTransaction(HiveTransactionManager transactionManager, HiveMetadata hiveMetadata) { this.transactionManager = requireNonNull(transactionManager, "transactionManager is null"); this.transactionHandle = new HiveTransactionHandle(); transactionManager.put(transactionHandle, hiveMetadata); getMetastore().testOnlyThrowOnCleanupFailures(); } @Override public ConnectorMetadata getMetadata() { return transactionManager.get(transactionHandle); } @Override public SemiTransactionalHiveMetastore getMetastore(String schema) { return getMetastore(); } private SemiTransactionalHiveMetastore getMetastore() { return ((HiveMetadata) transactionManager.get(transactionHandle)).getMetastore();
@Override public boolean isFinished() { if (!finished) { createSourceIfNecessary(); finished = (source != null) && source.isFinished(); if (source != null) { systemMemoryContext.setBytes(source.getSystemMemoryUsage()); } } return finished; }
@Override public void finish() { finished = true; blocked.set(null); if (source != null) { try { source.close(); } catch (IOException e) { throw Throwables.propagate(e); } systemMemoryContext.setBytes(source.getSystemMemoryUsage()); } }
@Override public boolean isFinished() { return delegate.isFinished(); }