@Override public Page getOutput() { Page page = pageSource.getNextPage(); if (page == null) { return null; } // update operator stats long endCompletedBytes = pageSource.getCompletedBytes(); long endReadTimeNanos = pageSource.getReadTimeNanos(); operatorContext.recordRawInputWithTiming(endCompletedBytes - completedBytes, endReadTimeNanos - readTimeNanos); operatorContext.recordProcessedInput(page.getSizeInBytes(), page.getPositionCount()); completedBytes = endCompletedBytes; readTimeNanos = endReadTimeNanos; // assure the page is in memory before handing to another operator page = page.getLoadedPage(); return page; }
@Benchmark public List<Page> read(CompressionCounter counter) throws IOException { if (!fileFormat.supports(data)) { throw new RuntimeException(fileFormat + " does not support data set " + dataSet); } List<Page> pages = new ArrayList<>(100); try (ConnectorPageSource pageSource = fileFormat.createFileFormatReader( SESSION, HDFS_ENVIRONMENT, dataFile, data.getColumnNames(), data.getColumnTypes())) { while (!pageSource.isFinished()) { Page page = pageSource.getNextPage(); if (page != null) { pages.add(page.getLoadedPage()); } } } counter.inputSize += data.getSize(); counter.outputSize += dataFile.length(); return pages; }
@Override public Page getOutput() { if (split == null) { return null; } if (source == null) { source = pageSourceProvider.createPageSource(operatorContext.getSession(), split, columns); } Page page = source.getNextPage(); if (page != null) { // assure the page is in memory before handing to another operator page = page.getLoadedPage(); // update operator stats long endCompletedBytes = source.getCompletedBytes(); long endReadTimeNanos = source.getReadTimeNanos(); operatorContext.recordRawInputWithTiming(endCompletedBytes - completedBytes, endReadTimeNanos - readTimeNanos); operatorContext.recordProcessedInput(page.getSizeInBytes(), page.getPositionCount()); completedBytes = endCompletedBytes; readTimeNanos = endReadTimeNanos; } // updating system memory usage should happen after page is loaded. systemMemoryContext.setBytes(source.getSystemMemoryUsage()); return page; } }
Page nextPage = pageSource.getNextPage(); if (nextPage != null) { pages.add(nextPage.getLoadedPage());
private static Page getNextPage(ConnectorPageSource pageSource) { Page page = null; while (isNullOrEmptyPage(page) && !pageSource.isFinished()) { page = pageSource.getNextPage(); if (page != null) { page = page.getLoadedPage(); } } return page; }
@Override public boolean hasNext() { if (hasMorePositions(currentPage, currentPosition)) { return true; } Page page = getNextPage(pageSource); if (isNullOrEmptyPage(page)) { return false; } currentPage = page.getLoadedPage(); currentPosition = 0; return true; }
@Benchmark public List<Page> read(CompressionCounter counter) throws IOException { if (!fileFormat.supports(data)) { throw new RuntimeException(fileFormat + " does not support data set " + dataSet); } List<Page> pages = new ArrayList<>(100); try (ConnectorPageSource pageSource = fileFormat.createFileFormatReader( SESSION, HDFS_ENVIRONMENT, dataFile, data.getColumnNames(), data.getColumnTypes())) { while (!pageSource.isFinished()) { Page page = pageSource.getNextPage(); if (page != null) { pages.add(page.getLoadedPage()); } } } counter.inputSize += data.getSize(); counter.outputSize += dataFile.length(); return pages; }
Page nextPage = pageSource.getNextPage(); if (nextPage != null) { pages.add(nextPage.getLoadedPage());