public AddDictionaryPageWork(Page page) { verify(canProcessDictionary(page), "invalid call to addDictionaryPage"); this.page = requireNonNull(page, "page is null"); this.dictionaryBlock = (DictionaryBlock) page.getBlock(channels[0]); updateDictionaryLookBack(dictionaryBlock.getDictionary()); this.dictionaryPage = createPageWithExtractedDictionary(page); }
@Test public void testCompact() { Slice[] expectedValues = createExpectedValues(5); DictionaryBlock dictionaryBlock = createDictionaryBlockWithUnreferencedKeys(expectedValues, 10); assertEquals(dictionaryBlock.isCompact(), false); DictionaryBlock compactBlock = dictionaryBlock.compact(); assertNotEquals(dictionaryBlock.getDictionarySourceId(), compactBlock.getDictionarySourceId()); assertEquals(compactBlock.getDictionary().getPositionCount(), (expectedValues.length / 2) + 1); assertBlock(compactBlock.getDictionary(), TestDictionaryBlock::createBlockBuilder, new Slice[] {expectedValues[0], expectedValues[1], expectedValues[3]}); assertDictionaryIds(compactBlock, 0, 1, 1, 2, 2, 0, 1, 1, 2, 2); assertEquals(compactBlock.isCompact(), true); DictionaryBlock reCompactedBlock = compactBlock.compact(); assertEquals(reCompactedBlock.getDictionarySourceId(), compactBlock.getDictionarySourceId()); }
public AddDictionaryPageWork(Page page) { verify(canProcessDictionary(page), "invalid call to addDictionaryPage"); this.page = requireNonNull(page, "page is null"); this.dictionaryBlock = (DictionaryBlock) page.getBlock(channels[0]); updateDictionaryLookBack(dictionaryBlock.getDictionary()); this.dictionaryPage = createPageWithExtractedDictionary(page); }
private Page createPageWithExtractedDictionary(Page page) { Block[] blocks = new Block[page.getChannelCount()]; Block dictionary = ((DictionaryBlock) page.getBlock(channels[0])).getDictionary(); // extract data dictionary blocks[channels[0]] = dictionary; // extract hash dictionary if (inputHashChannel.isPresent()) { blocks[inputHashChannel.get()] = ((DictionaryBlock) page.getBlock(inputHashChannel.get())).getDictionary(); } return new Page(dictionary.getPositionCount(), blocks); }
private Page createPageWithExtractedDictionary(Page page) { Block[] blocks = new Block[page.getChannelCount()]; Block dictionary = ((DictionaryBlock) page.getBlock(channels[0])).getDictionary(); // extract data dictionary blocks[channels[0]] = dictionary; // extract hash dictionary if (inputHashChannel.isPresent()) { blocks[inputHashChannel.get()] = ((DictionaryBlock) page.getBlock(inputHashChannel.get())).getDictionary(); } return new Page(dictionary.getPositionCount(), blocks); }
public GetDictionaryGroupIdsWork(Page page) { this.page = requireNonNull(page, "page is null"); verify(canProcessDictionary(page), "invalid call to processDictionary"); this.dictionaryBlock = (DictionaryBlock) page.getBlock(channels[0]); updateDictionaryLookBack(dictionaryBlock.getDictionary()); this.dictionaryPage = createPageWithExtractedDictionary(page); // we know the exact size required for the block this.blockBuilder = BIGINT.createFixedSizeBlockBuilder(page.getPositionCount()); }
public GetDictionaryGroupIdsWork(Page page) { this.page = requireNonNull(page, "page is null"); verify(canProcessDictionary(page), "invalid call to processDictionary"); this.dictionaryBlock = (DictionaryBlock) page.getBlock(channels[0]); updateDictionaryLookBack(dictionaryBlock.getDictionary()); this.dictionaryPage = createPageWithExtractedDictionary(page); // we know the exact size required for the block this.blockBuilder = BIGINT.createFixedSizeBlockBuilder(page.getPositionCount()); }
@Test public void testCopyPositionsWithCompactionsAndReorder() { Slice[] expectedValues = createExpectedValues(10); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); int[] positionsToCopy = new int[] {50, 55, 40, 45, 60}; DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy, 0, positionsToCopy.length); assertEquals(copiedBlock.getDictionary().getPositionCount(), 2); assertEquals(copiedBlock.getPositionCount(), positionsToCopy.length); assertBlock(copiedBlock.getDictionary(), TestDictionaryBlock::createBlockBuilder, new Slice[] {expectedValues[0], expectedValues[5]}); assertDictionaryIds(copiedBlock, 0, 1, 0, 1, 0); }
@Test public void testSizeInBytes() { Slice[] expectedValues = createExpectedValues(10); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); assertEquals(dictionaryBlock.getSizeInBytes(), dictionaryBlock.getDictionary().getSizeInBytes() + (100 * SIZE_OF_INT)); }
@Test public void testCopyPositionsSamePosition() { Slice[] expectedValues = createExpectedValues(10); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); int[] positionsToCopy = new int[] {52, 52, 52}; DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy, 0, positionsToCopy.length); assertEquals(copiedBlock.getDictionary().getPositionCount(), 1); assertEquals(copiedBlock.getPositionCount(), positionsToCopy.length); assertBlock(copiedBlock.getDictionary(), TestDictionaryBlock::createBlockBuilder, new Slice[] {expectedValues[2]}); assertDictionaryIds(copiedBlock, 0, 0, 0); }
@Test public void testCopyPositionsWithCompactionsAndReorder() { Slice[] expectedValues = createExpectedValues(10); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); int[] positionsToCopy = new int[] {50, 55, 40, 45, 60}; DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy, 0, positionsToCopy.length); assertEquals(copiedBlock.getDictionary().getPositionCount(), 2); assertEquals(copiedBlock.getPositionCount(), positionsToCopy.length); assertBlock(copiedBlock.getDictionary(), TestDictionaryBlock::createBlockBuilder, new Slice[] {expectedValues[0], expectedValues[5]}); assertDictionaryIds(copiedBlock, 0, 1, 0, 1, 0); }
@Test public void testCopyPositionsSamePosition() { Slice[] expectedValues = createExpectedValues(10); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); int[] positionsToCopy = new int[] {52, 52, 52}; DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy, 0, positionsToCopy.length); assertEquals(copiedBlock.getDictionary().getPositionCount(), 1); assertEquals(copiedBlock.getPositionCount(), positionsToCopy.length); assertBlock(copiedBlock.getDictionary(), TestDictionaryBlock::createBlockBuilder, new Slice[] {expectedValues[2]}); assertDictionaryIds(copiedBlock, 0, 0, 0); }
@Test public void testCopyPositionsWithCompaction() { Slice[] expectedValues = createExpectedValues(10); Slice firstExpectedValue = expectedValues[0]; DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); int[] positionsToCopy = new int[] {0, 10, 20, 30, 40}; DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy, 0, positionsToCopy.length); assertEquals(copiedBlock.getDictionary().getPositionCount(), 1); assertEquals(copiedBlock.getPositionCount(), positionsToCopy.length); assertBlock(copiedBlock.getDictionary(), TestDictionaryBlock::createBlockBuilder, new Slice[] {firstExpectedValue}); assertBlock(copiedBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] {firstExpectedValue, firstExpectedValue, firstExpectedValue, firstExpectedValue, firstExpectedValue}); }
@Test public void testSizeInBytes() { Slice[] expectedValues = createExpectedValues(10); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); assertEquals(dictionaryBlock.getSizeInBytes(), dictionaryBlock.getDictionary().getSizeInBytes() + (100 * SIZE_OF_INT)); }
@Test public void testCopyPositionsWithCompaction() { Slice[] expectedValues = createExpectedValues(10); Slice firstExpectedValue = expectedValues[0]; DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); int[] positionsToCopy = new int[] {0, 10, 20, 30, 40}; DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy, 0, positionsToCopy.length); assertEquals(copiedBlock.getDictionary().getPositionCount(), 1); assertEquals(copiedBlock.getPositionCount(), positionsToCopy.length); assertBlock(copiedBlock.getDictionary(), TestDictionaryBlock::createBlockBuilder, new Slice[] {firstExpectedValue}); assertBlock(copiedBlock, TestDictionaryBlock::createBlockBuilder, new Slice[] {firstExpectedValue, firstExpectedValue, firstExpectedValue, firstExpectedValue, firstExpectedValue}); }
@Test public void testCompactAllKeysReferenced() { Slice[] expectedValues = createExpectedValues(5); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 10); DictionaryBlock compactBlock = dictionaryBlock.compact(); // When there is nothing to compact, we return the same block assertEquals(compactBlock.getDictionary(), dictionaryBlock.getDictionary()); assertEquals(compactBlock.getPositionCount(), dictionaryBlock.getPositionCount()); for (int position = 0; position < compactBlock.getPositionCount(); position++) { assertEquals(compactBlock.getId(position), dictionaryBlock.getId(position)); } assertEquals(compactBlock.isCompact(), true); }
@Test public void testCompactAllKeysReferenced() { Slice[] expectedValues = createExpectedValues(5); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 10); DictionaryBlock compactBlock = dictionaryBlock.compact(); // When there is nothing to compact, we return the same block assertEquals(compactBlock.getDictionary(), dictionaryBlock.getDictionary()); assertEquals(compactBlock.getPositionCount(), dictionaryBlock.getPositionCount()); for (int position = 0; position < compactBlock.getPositionCount(); position++) { assertEquals(compactBlock.getId(position), dictionaryBlock.getId(position)); } assertEquals(compactBlock.isCompact(), true); }
@Test public void testCopyPositionsNoCompaction() { Slice[] expectedValues = createExpectedValues(1); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); int[] positionsToCopy = new int[] {0, 2, 4, 5}; DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy, 0, positionsToCopy.length); assertEquals(copiedBlock.getPositionCount(), positionsToCopy.length); assertBlock(copiedBlock.getDictionary(), TestDictionaryBlock::createBlockBuilder, expectedValues); }
@Test public void testCopyPositionsNoCompaction() { Slice[] expectedValues = createExpectedValues(1); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); int[] positionsToCopy = new int[] {0, 2, 4, 5}; DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy, 0, positionsToCopy.length); assertEquals(copiedBlock.getPositionCount(), positionsToCopy.length); assertBlock(copiedBlock.getDictionary(), TestDictionaryBlock::createBlockBuilder, expectedValues); }
@Test public void testSanityColumnarDictionary() { PageProcessor processor = compiler.compilePageProcessor(Optional.empty(), ImmutableList.of(field(0, VARCHAR)), MAX_BATCH_SIZE).get(); Page page = new Page(createDictionaryBlock(createExpectedValues(10), 100)); Page outputPage = getOnlyElement( processor.process( null, new DriverYieldSignal(), newSimpleAggregatedMemoryContext().newLocalMemoryContext(PageProcessor.class.getSimpleName()), page)) .orElseThrow(() -> new AssertionError("page is not present")); assertEquals(outputPage.getPositionCount(), 100); assertTrue(outputPage.getBlock(0) instanceof DictionaryBlock); DictionaryBlock dictionaryBlock = (DictionaryBlock) outputPage.getBlock(0); assertEquals(dictionaryBlock.getDictionary().getPositionCount(), 10); }