public Work<?> addPage(Page page) { // Just add the page to the pending work, which will be processed later. return hash.addPage(page); }
@Override public Work<?> processPage(Page page) { if (aggregators.isEmpty()) { return groupByHash.addPage(page); } else { return new TransformWork<>( groupByHash.getGroupIds(page), groupByIdBlock -> { for (Aggregator aggregator : aggregators) { aggregator.processPage(groupByIdBlock, page); } // we do not need any output from TransformWork for this case return null; }); } }
@Test public void testNullGroup() { GroupByHash groupByHash = createGroupByHash(TEST_SESSION, ImmutableList.of(BIGINT), new int[] {0}, Optional.of(1), 100, JOIN_COMPILER); Block block = createLongsBlock((Long) null); Block hashBlock = getHashBlock(ImmutableList.of(BIGINT), block); Page page = new Page(block, hashBlock); groupByHash.addPage(page).process(); // Add enough values to force a rehash block = createLongSequenceBlock(1, 132748); hashBlock = getHashBlock(ImmutableList.of(BIGINT), block); page = new Page(block, hashBlock); groupByHash.addPage(page).process(); block = createLongsBlock(0); hashBlock = getHashBlock(ImmutableList.of(BIGINT), block); page = new Page(block, hashBlock); assertFalse(groupByHash.contains(0, page, CONTAINS_CHANNELS)); }
@Test public void testAddPage() { GroupByHash groupByHash = createGroupByHash(TEST_SESSION, ImmutableList.of(BIGINT), new int[] {0}, Optional.of(1), 100, JOIN_COMPILER); for (int tries = 0; tries < 2; tries++) { for (int value = 0; value < MAX_GROUP_ID; value++) { Block block = BlockAssertions.createLongsBlock(value); Block hashBlock = TypeUtils.getHashBlock(ImmutableList.of(BIGINT), block); Page page = new Page(block, hashBlock); for (int addValuesTries = 0; addValuesTries < 10; addValuesTries++) { groupByHash.addPage(page).process(); assertEquals(groupByHash.getGroupCount(), tries == 0 ? value + 1 : MAX_GROUP_ID); // add the page again using get group ids and make sure the group count didn't change Work<GroupByIdBlock> work = groupByHash.getGroupIds(page); work.process(); GroupByIdBlock groupIds = work.getResult(); assertEquals(groupByHash.getGroupCount(), tries == 0 ? value + 1 : MAX_GROUP_ID); assertEquals(groupIds.getGroupCount(), tries == 0 ? value + 1 : MAX_GROUP_ID); // verify the first position assertEquals(groupIds.getPositionCount(), 1); long groupId = groupIds.getGroupId(0); assertEquals(groupId, value); } } } }
@Benchmark @OperationsPerInvocation(POSITIONS) public Object bigintGroupByHash(SingleChannelBenchmarkData data) { GroupByHash groupByHash = new BigintGroupByHash(0, data.getHashEnabled(), EXPECTED_SIZE, NOOP); data.getPages().forEach(p -> groupByHash.addPage(p).process()); ImmutableList.Builder<Page> pages = ImmutableList.builder(); PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes()); for (int groupId = 0; groupId < groupByHash.getGroupCount(); groupId++) { pageBuilder.declarePosition(); groupByHash.appendValuesTo(groupId, pageBuilder, 0); if (pageBuilder.isFull()) { pages.add(pageBuilder.build()); pageBuilder.reset(); } } pages.add(pageBuilder.build()); return pageBuilder.build(); }
@Benchmark @OperationsPerInvocation(POSITIONS) public Object addPagePreCompute(BenchmarkData data) { GroupByHash groupByHash = new MultiChannelGroupByHash(data.getTypes(), data.getChannels(), data.getHashChannel(), EXPECTED_SIZE, false, getJoinCompiler(data.isGroupByUsesEqual()), NOOP); data.getPages().forEach(p -> groupByHash.addPage(p).process()); ImmutableList.Builder<Page> pages = ImmutableList.builder(); PageBuilder pageBuilder = new PageBuilder(groupByHash.getTypes()); for (int groupId = 0; groupId < groupByHash.getGroupCount(); groupId++) { pageBuilder.declarePosition(); groupByHash.appendValuesTo(groupId, pageBuilder, 0); if (pageBuilder.isFull()) { pages.add(pageBuilder.build()); pageBuilder.reset(); } } pages.add(pageBuilder.build()); return pageBuilder.build(); }
private void processPage(Page page) { if (aggregators.isEmpty()) { groupByHash.addPage(page); return; } GroupByIdBlock groupIds = groupByHash.getGroupIds(page); for (Aggregator aggregator : aggregators) { aggregator.processPage(groupIds, page); } }
public void addPage(Page page) { hash.addPage(page); if (operatorContext != null) { operatorContext.setMemoryReservation(hash.getEstimatedSize()); } } }
@Test public void testNullGroup() throws Exception { GroupByHash groupByHash = createGroupByHash(TEST_SESSION, ImmutableList.of(BIGINT), new int[] { 0}, Optional.<Integer>empty(), Optional.of(1), 100); Block block = createLongsBlock((Long) null); Block hashBlock = getHashBlock(ImmutableList.of(BIGINT), block); Page page = new Page(block, hashBlock); groupByHash.addPage(page); // Add enough values to force a rehash block = createLongSequenceBlock(1, 132748); hashBlock = getHashBlock(ImmutableList.of(BIGINT), block); page = new Page(block, hashBlock); groupByHash.addPage(page); block = createLongsBlock(0); hashBlock = getHashBlock(ImmutableList.of(BIGINT), block); page = new Page(block, hashBlock); assertFalse(groupByHash.contains(0, page, CONTAINS_CHANNELS)); }
@Test public void testAddPage() throws Exception { GroupByHash groupByHash = createGroupByHash(TEST_SESSION, ImmutableList.of(BIGINT), new int[] { 0}, Optional.<Integer>empty(), Optional.of(1), 100); for (int tries = 0; tries < 2; tries++) { for (int value = 0; value < MAX_GROUP_ID; value++) { Block block = BlockAssertions.createLongsBlock(value); Block hashBlock = TypeUtils.getHashBlock(ImmutableList.of(BIGINT), block); Page page = new Page(block, hashBlock); for (int addValuesTries = 0; addValuesTries < 10; addValuesTries++) { groupByHash.addPage(page); assertEquals(groupByHash.getGroupCount(), tries == 0 ? value + 1 : MAX_GROUP_ID); // add the page again using get group ids and make sure the group count didn't change GroupByIdBlock groupIds = groupByHash.getGroupIds(page); assertEquals(groupByHash.getGroupCount(), tries == 0 ? value + 1 : MAX_GROUP_ID); assertEquals(groupIds.getGroupCount(), tries == 0 ? value + 1 : MAX_GROUP_ID); // verify the first position assertEquals(groupIds.getPositionCount(), 1); long groupId = groupIds.getGroupId(0); assertEquals(groupId, value); } } } }