@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { // The down casts here are safe because it is the block itself the provides this encoding implementation. DictionaryBlock dictionaryBlock = (DictionaryBlock) block; dictionaryBlock = dictionaryBlock.compact(); // positionCount int positionCount = dictionaryBlock.getPositionCount(); sliceOutput.appendInt(positionCount); // dictionary Block dictionary = dictionaryBlock.getDictionary(); blockEncodingSerde.writeBlock(sliceOutput, dictionary); // ids sliceOutput.writeBytes(dictionaryBlock.getIds()); // instance id sliceOutput.appendLong(dictionaryBlock.getDictionarySourceId().getMostSignificantBits()); sliceOutput.appendLong(dictionaryBlock.getDictionarySourceId().getLeastSignificantBits()); sliceOutput.appendLong(dictionaryBlock.getDictionarySourceId().getSequenceId()); }
private static int[] filterIds(ProjectionFunction projection, Page page, int[] selectedPositions) { Slice ids = ((DictionaryBlock) page.getBlock(getOnlyElement(projection.getInputChannels()))).getIds(); int[] outputIds = new int[selectedPositions.length]; for (int pos = 0; pos < selectedPositions.length; pos++) { outputIds[pos] = ids.getInt(selectedPositions[pos] * SizeOf.SIZE_OF_INT); } return outputIds; }
@Test public void testCompactAllKeysReferenced() throws Exception { Slice[] expectedValues = createExpectedValues(5); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 10); DictionaryBlock compactBlock = dictionaryBlock.compact(); // When there is nothing to compact, we return the same block assertEquals(compactBlock.getDictionary(), dictionaryBlock.getDictionary()); assertEquals(compactBlock.getIds(), dictionaryBlock.getIds()); assertEquals(compactBlock.isCompact(), true); }
@Test public void testCopyPositionsWithCompactionsAndReorder() throws Exception { Slice[] expectedValues = createExpectedValues(10); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); List<Integer> positionsToCopy = Ints.asList(50, 55, 40, 45, 60); DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy); assertEquals(copiedBlock.getDictionary().getPositionCount(), 2); assertEquals(copiedBlock.getPositionCount(), positionsToCopy.size()); assertBlock(copiedBlock.getDictionary(), new Slice[] { expectedValues[0], expectedValues[5] }); assertEquals(copiedBlock.getIds(), wrappedIntArray(0, 1, 0, 1, 0)); }
@Test public void testCopyPositionsSamePosition() throws Exception { Slice[] expectedValues = createExpectedValues(10); DictionaryBlock dictionaryBlock = createDictionaryBlock(expectedValues, 100); List<Integer> positionsToCopy = Ints.asList(52, 52, 52); DictionaryBlock copiedBlock = (DictionaryBlock) dictionaryBlock.copyPositions(positionsToCopy); assertEquals(copiedBlock.getDictionary().getPositionCount(), 1); assertEquals(copiedBlock.getPositionCount(), positionsToCopy.size()); assertBlock(copiedBlock.getDictionary(), new Slice[] { expectedValues[2] }); assertEquals(copiedBlock.getIds(), wrappedIntArray(0, 0, 0)); }
@Test public void testDictionaryBlockGetRegion() throws Exception { Slice[] expectedValues = createExpectedValues(3); int[] ids = new int[] { 0, 2, 1, 0, 0, 0, 1, 1, 1, 0, 1, 2 }; boolean[] isNull = new boolean[ids.length]; isNull[2] = true; LazyBlockLoader<LazySliceArrayBlock> loader = new TestDictionaryLazySliceArrayBlockLoader(expectedValues, ids, isNull); LazySliceArrayBlock block = new LazySliceArrayBlock(ids.length, loader); Block region = block.getRegion(0, 3); assertFalse(region.isNull(0)); assertFalse(region.isNull(1)); assertTrue(region.isNull(2)); assertTrue(region instanceof DictionaryBlock); DictionaryBlock dictionaryBlock = (DictionaryBlock) region; assertEquals(((SliceArrayBlock) dictionaryBlock.getDictionary()).getValues(), new Slice[] { expectedValues[0], expectedValues[2], null }); // The values in the dictionary are rearranged during compaction in the order in which they are referenced, // with a null appended to the end of the list if applicable assertEquals(dictionaryBlock.getIds(), Slices.wrappedIntArray(0, 1, 2)); }
@Test public void testCompact() throws Exception { Slice[] expectedValues = createExpectedValues(5); DictionaryBlock dictionaryBlock = createDictionaryBlockWithUnreferencedKeys(expectedValues, 10); assertEquals(dictionaryBlock.isCompact(), false); DictionaryBlock compactBlock = dictionaryBlock.compact(); assertNotEquals(dictionaryBlock.getDictionarySourceId(), compactBlock.getDictionarySourceId()); assertEquals(compactBlock.getDictionary().getPositionCount(), (expectedValues.length / 2) + 1); assertBlock(compactBlock.getDictionary(), new Slice[] { expectedValues[0], expectedValues[1], expectedValues[3] }); assertEquals(compactBlock.getIds(), wrappedIntArray(0, 1, 1, 2, 2, 0, 1, 1, 2, 2)); assertEquals(compactBlock.isCompact(), true); DictionaryBlock reCompactedBlock = compactBlock.compact(); assertEquals(reCompactedBlock.getDictionarySourceId(), compactBlock.getDictionarySourceId()); }
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { // The down casts here are safe because it is the block itself the provides this encoding implementation. DictionaryBlock dictionaryBlock = (DictionaryBlock) block; dictionaryBlock = dictionaryBlock.compact(); // positionCount int positionCount = dictionaryBlock.getPositionCount(); sliceOutput.appendInt(positionCount); // dictionary Block dictionary = dictionaryBlock.getDictionary(); blockEncodingSerde.writeBlock(sliceOutput, dictionary); // ids sliceOutput.writeBytes(dictionaryBlock.getIds()); // instance id sliceOutput.appendLong(dictionaryBlock.getDictionarySourceId().getMostSignificantBits()); sliceOutput.appendLong(dictionaryBlock.getDictionarySourceId().getLeastSignificantBits()); sliceOutput.appendLong(dictionaryBlock.getDictionarySourceId().getSequenceId()); }