private void addRowsToAggregates(Page page, int startPosition, int endPosition) { for (Aggregator aggregator : aggregates) { aggregator.processPage(page.getRegion(startPosition, endPosition - startPosition + 1)); } }
public IndexedData getIndexedDataForKeys(int position, Page indexPage) { // Normalize the indexBlocks so that they only encompass the unloaded positions int totalPositions = indexPage.getPositionCount(); int remainingPositions = totalPositions - position; return getIndexedDataForKeys(indexPage.getRegion(position, remainingPositions)); }
private static List<Page> splitPage(Page page, long maxPageSizeInBytes, long previousPageSize) { checkArgument(page.getPositionCount() > 0, "page is empty"); checkArgument(maxPageSizeInBytes > 0, "maxPageSizeInBytes must be > 0"); // for Pages with certain types of Blocks (e.g., RLE blocks) the size in bytes may remain constant // through the recursive calls, which causes the recursion to only terminate when page.getPositionCount() == 1 // and create potentially a large number of Page's of size 1. So we check here that // if the size of the page doesn't improve from the previous call we terminate the recursion. if (page.getSizeInBytes() == previousPageSize || page.getSizeInBytes() <= maxPageSizeInBytes || page.getPositionCount() == 1) { return ImmutableList.of(page); } ImmutableList.Builder<Page> outputPages = ImmutableList.builder(); long previousSize = page.getSizeInBytes(); int positionCount = page.getPositionCount(); int half = positionCount / 2; Page leftHalf = page.getRegion(0, half); outputPages.addAll(splitPage(leftHalf, maxPageSizeInBytes, previousSize)); Page rightHalf = page.getRegion(half, positionCount - half); outputPages.addAll(splitPage(rightHalf, maxPageSizeInBytes, previousSize)); return outputPages.build(); } }
private Page pageTail(Page currentPage, int startAtPosition) { verify(currentPage.getPositionCount() - startAtPosition >= 0); return currentPage.getRegion(startAtPosition, currentPage.getPositionCount() - startAtPosition); }
@Test(expectedExceptions = IndexOutOfBoundsException.class, expectedExceptionsMessageRegExp = "Invalid position 1 and length 1 in page with 0 positions") public void testGetRegionExceptions() { new Page(0).getRegion(1, 1); }
private static List<Page> splitPage(Page page, long maxPageSizeInBytes, long previousPageSize) { checkArgument(page.getPositionCount() > 0, "page is empty"); checkArgument(maxPageSizeInBytes > 0, "maxPageSizeInBytes must be > 0"); // for Pages with certain types of Blocks (e.g., RLE blocks) the size in bytes may remain constant // through the recursive calls, which causes the recursion to only terminate when page.getPositionCount() == 1 // and create potentially a large number of Page's of size 1. So we check here that // if the size of the page doesn't improve from the previous call we terminate the recursion. if (page.getSizeInBytes() == previousPageSize || page.getSizeInBytes() <= maxPageSizeInBytes || page.getPositionCount() == 1) { return ImmutableList.of(page); } ImmutableList.Builder<Page> outputPages = ImmutableList.builder(); long previousSize = page.getSizeInBytes(); int positionCount = page.getPositionCount(); int half = positionCount / 2; Page leftHalf = page.getRegion(0, half); outputPages.addAll(splitPage(leftHalf, maxPageSizeInBytes, previousSize)); Page rightHalf = page.getRegion(half, positionCount - half); outputPages.addAll(splitPage(rightHalf, maxPageSizeInBytes, previousSize)); return outputPages.build(); } }
@Test public void testGetRegion() { assertEquals(new Page(10).getRegion(5, 5).getPositionCount(), 5); }
@Test public void testGetRegionFromNoColumnPage() { assertEquals(new Page(100).getRegion(0, 10).getPositionCount(), 10); }
/** * @return the unused section of the page, or null if fully applied. * pagesIndex guaranteed to have at least one row after this method returns */ private Page updatePagesIndex(Page page) { checkArgument(page.getPositionCount() > 0); // TODO: Fix pagesHashStrategy to allow specifying channels for comparison, it currently requires us to rearrange the right side blocks in consecutive channel order Page preGroupedPage = rearrangePage(page, preGroupedChannels); if (pagesIndex.getPositionCount() == 0 || pagesIndex.positionEqualsRow(preGroupedPartitionHashStrategy, 0, 0, preGroupedPage)) { // Find the position where the pre-grouped columns change int groupEnd = findGroupEnd(preGroupedPage, preGroupedPartitionHashStrategy, 0); // Add the section of the page that contains values for the current group pagesIndex.addPage(page.getRegion(0, groupEnd)); if (page.getPositionCount() - groupEnd > 0) { // Save the remaining page, which may contain multiple partitions return page.getRegion(groupEnd, page.getPositionCount() - groupEnd); } else { // Page fully consumed return null; } } else { // We had previous results buffered, but the new page starts with new group values return page; } }
int attemptedPositions = totalPositions / 10; while (attemptedPositions > 1) { myUpdateRequest = new UpdateRequest(indexPage.getRegion(0, attemptedPositions)); if (indexSnapshotLoader.load(ImmutableList.of(myUpdateRequest))) { stats.recordSuccessfulIndexJoinLookupByLimitedRequest();
public IndexedData streamIndexDataForSingleKey(UpdateRequest updateRequest) { Page indexKeyTuple = updateRequest.getPage().getRegion(0, 1); PageBuffer pageBuffer = new PageBuffer(100); DriverFactory driverFactory = indexBuildDriverFactoryProvider.createStreaming(pageBuffer, indexKeyTuple); Driver driver = driverFactory.createDriver(pipelineContext.addDriverContext()); PageRecordSet pageRecordSet = new PageRecordSet(keyTypes, indexKeyTuple); PlanNodeId planNodeId = driverFactory.getSourceId().get(); ScheduledSplit split = new ScheduledSplit(0, planNodeId, new Split(INDEX_CONNECTOR_ID, new ConnectorTransactionHandle() {}, new IndexSplit(pageRecordSet))); driver.updateSource(new TaskSource(planNodeId, ImmutableSet.of(split), true)); return new StreamingIndexedData(outputTypes, keyTypes, indexKeyTuple, pageBuffer, driver); }
public static void assertAggregation(InternalAggregationFunction function, BiFunction<Object, Object, Boolean> equalAssertion, String testDescription, Page page, Object expectedValue) { int positions = page.getPositionCount(); for (int i = 1; i < page.getChannelCount(); i++) { assertEquals(positions, page.getBlock(i).getPositionCount(), "input blocks provided are not equal in position count"); } if (positions == 0) { assertAggregationInternal(function, equalAssertion, testDescription, expectedValue, new Page[] {}); } else if (positions == 1) { assertAggregationInternal(function, equalAssertion, testDescription, expectedValue, page); } else { int split = positions / 2; // [0, split - 1] goes to first list of blocks; [split, positions - 1] goes to second list of blocks. Page page1 = page.getRegion(0, split); Page page2 = page.getRegion(split, positions - split); assertAggregationInternal(function, equalAssertion, testDescription, expectedValue, page1, page2); } }
private void processInput(Page page) { requireNonNull(page, "page is null"); Page groupByPage = extractColumns(page, groupByChannels); if (currentGroup != null) { if (!pagesHashStrategy.rowEqualsRow(0, extractColumns(currentGroup, groupByChannels), 0, groupByPage)) { // page starts with new group, so flush it evaluateAndFlushGroup(currentGroup, 0); } currentGroup = null; } int startPosition = 0; while (true) { // may be equal to page.getPositionCount() if the end is not found in this page int nextGroupStart = findNextGroupStart(startPosition, groupByPage); addRowsToAggregates(page, startPosition, nextGroupStart - 1); if (nextGroupStart < page.getPositionCount()) { // current group stops somewhere in the middle of the page, so flush it evaluateAndFlushGroup(page, startPosition); startPosition = nextGroupStart; } else { currentGroup = page.getRegion(page.getPositionCount() - 1, 1); return; } } }
public static List<Page> splitPage(Page page, long maxPageSizeInBytes) { checkArgument(page.getPositionCount() > 0, "page is empty"); checkArgument(maxPageSizeInBytes > 0, "maxPageSizeInBytes must be > 0"); if (page.getSizeInBytes() <= maxPageSizeInBytes || page.getPositionCount() == 1) { return ImmutableList.of(page); } ImmutableList.Builder<Page> outputPages = ImmutableList.builder(); int positionCount = page.getPositionCount(); int half = positionCount / 2; Page splitPage1 = page.getRegion(0, half); outputPages.addAll(splitPage(splitPage1, maxPageSizeInBytes)); Page splitPage2 = page.getRegion(half, positionCount - half); outputPages.addAll(splitPage(splitPage2, maxPageSizeInBytes)); return outputPages.build(); } }
private static List<Page> splitPage(Page page, long maxPageSizeInBytes, long previousPageSize) { checkArgument(page.getPositionCount() > 0, "page is empty"); checkArgument(maxPageSizeInBytes > 0, "maxPageSizeInBytes must be > 0"); // for Pages with certain types of Blocks (e.g., RLE blocks) the size in bytes may remain constant // through the recursive calls, which causes the recursion to only terminate when page.getPositionCount() == 1 // and create potentially a large number of Page's of size 1. So we check here that // if the size of the page doesn't improve from the previous call we terminate the recursion. if (page.getSizeInBytes() == previousPageSize || page.getSizeInBytes() <= maxPageSizeInBytes || page.getPositionCount() == 1) { return ImmutableList.of(page); } ImmutableList.Builder<Page> outputPages = ImmutableList.builder(); long previousSize = page.getSizeInBytes(); int positionCount = page.getPositionCount(); int half = positionCount / 2; Page leftHalf = page.getRegion(0, half); outputPages.addAll(splitPage(leftHalf, maxPageSizeInBytes, previousSize)); Page rightHalf = page.getRegion(half, positionCount - half); outputPages.addAll(splitPage(rightHalf, maxPageSizeInBytes, previousSize)); return outputPages.build(); } }
/** * @return the unused section of the page, or null if fully applied. * pagesIndex guaranteed to have at least one row after this method returns */ private Page updatePagesIndex(Page page) { checkArgument(page.getPositionCount() > 0); // TODO: Fix pagesHashStrategy to allow specifying channels for comparison, it currently requires us to rearrange the right side blocks in consecutive channel order Page preGroupedPage = rearrangePage(page, preGroupedChannels); if (pagesIndex.getPositionCount() == 0 || pagesIndex.positionEqualsRow(preGroupedPartitionHashStrategy, 0, 0, preGroupedPage.getBlocks())) { // Find the position where the pre-grouped columns change int groupEnd = findGroupEnd(preGroupedPage, preGroupedPartitionHashStrategy, 0); // Add the section of the page that contains values for the current group pagesIndex.addPage(page.getRegion(0, groupEnd)); if (page.getPositionCount() - groupEnd > 0) { // Save the remaining page, which may contain multiple partitions return page.getRegion(groupEnd, page.getPositionCount() - groupEnd); } else { // Page fully consumed return null; } } else { // We had previous results buffered, but the new page starts with new group values return page; } }