public DictionaryBlock(int positionCount, Block dictionary, int[] ids) { this(0, positionCount, dictionary, ids, false, randomDictionaryId()); }
private boolean canProcessDictionary(Page page) { if (!this.processDictionary || channels.length > 1 || !(page.getBlock(channels[0]) instanceof DictionaryBlock)) { return false; } if (inputHashChannel.isPresent()) { Block inputHashBlock = page.getBlock(inputHashChannel.get()); DictionaryBlock inputDataBlock = (DictionaryBlock) page.getBlock(channels[0]); if (!(inputHashBlock instanceof DictionaryBlock)) { // data channel is dictionary encoded but hash channel is not return false; } if (!((DictionaryBlock) inputHashBlock).getDictionarySourceId().equals(inputDataBlock.getDictionarySourceId())) { // dictionarySourceIds of data block and hash block do not match return false; } } return true; }
public static DictionaryId randomDictionaryId() { return new DictionaryId(nodeId.getMostSignificantBits(), nodeId.getLeastSignificantBits(), sequenceGenerator.getAndIncrement()); }
DictionaryId newDictionaryId = randomDictionaryId(); for (DictionaryBlock dictionaryBlock : blocks) { if (!firstDictionaryBlock.getDictionarySourceId().equals(dictionaryBlock.getDictionarySourceId())) { throw new IllegalArgumentException("dictionarySourceIds must be the same");
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { // The down casts here are safe because it is the block itself the provides this encoding implementation. DictionaryBlock dictionaryBlock = (DictionaryBlock) block; dictionaryBlock = dictionaryBlock.compact(); // positionCount int positionCount = dictionaryBlock.getPositionCount(); sliceOutput.appendInt(positionCount); // dictionary Block dictionary = dictionaryBlock.getDictionary(); blockEncodingSerde.writeBlock(sliceOutput, dictionary); // ids sliceOutput.writeBytes(dictionaryBlock.getIds()); // instance id sliceOutput.appendLong(dictionaryBlock.getDictionarySourceId().getMostSignificantBits()); sliceOutput.appendLong(dictionaryBlock.getDictionarySourceId().getLeastSignificantBits()); sliceOutput.appendLong(dictionaryBlock.getDictionarySourceId().getSequenceId()); }
DictionaryId newDictionaryId = randomDictionaryId(); for (DictionaryBlock dictionaryBlock : blocks) { if (!firstDictionaryBlock.getDictionarySourceId().equals(dictionaryBlock.getDictionarySourceId())) { throw new IllegalArgumentException("dictionarySourceIds must be the same");
@Override public void writeBlock(BlockEncodingSerde blockEncodingSerde, SliceOutput sliceOutput, Block block) { // The down casts here are safe because it is the block itself the provides this encoding implementation. DictionaryBlock dictionaryBlock = (DictionaryBlock) block; dictionaryBlock = dictionaryBlock.compact(); // positionCount int positionCount = dictionaryBlock.getPositionCount(); sliceOutput.appendInt(positionCount); // dictionary Block dictionary = dictionaryBlock.getDictionary(); blockEncodingSerde.writeBlock(sliceOutput, dictionary); // ids sliceOutput.writeBytes(dictionaryBlock.getIds()); // instance id sliceOutput.appendLong(dictionaryBlock.getDictionarySourceId().getMostSignificantBits()); sliceOutput.appendLong(dictionaryBlock.getDictionarySourceId().getLeastSignificantBits()); sliceOutput.appendLong(dictionaryBlock.getDictionarySourceId().getSequenceId()); }
public DictionaryBlock(int positionCount, Block dictionary, int[] ids, boolean dictionaryIsCompacted) { this(0, positionCount, dictionary, ids, dictionaryIsCompacted, randomDictionaryId()); }
private boolean canProcessDictionary(Page page) { boolean processDictionary = this.processDictionary && maskChannel < 0 && channels.length == 1 && page.getBlock(channels[0]) instanceof DictionaryBlock; if (processDictionary && inputHashChannel.isPresent()) { Block inputHashBlock = page.getBlock(inputHashChannel.get()); DictionaryBlock inputDataBlock = (DictionaryBlock) page.getBlock(channels[0]); verify(inputHashBlock instanceof DictionaryBlock, "data channel is dictionary encoded but hash channel is not"); verify(((DictionaryBlock) inputHashBlock).getDictionarySourceId().equals(inputDataBlock.getDictionarySourceId()), "dictionarySourceIds of data block and hash block do not match"); } return processDictionary; }
@Override public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { // positionCount int positionCount = sliceInput.readInt(); // dictionary Block dictionaryBlock = blockEncodingSerde.readBlock(sliceInput); // ids int[] ids = new int[positionCount]; sliceInput.readBytes(Slices.wrappedIntArray(ids)); // instance id long mostSignificantBits = sliceInput.readLong(); long leastSignificantBits = sliceInput.readLong(); long sequenceId = sliceInput.readLong(); // We always compact the dictionary before we send it. However, dictionaryBlock comes from sliceInput, which may over-retain memory. // As a result, setting dictionaryIsCompacted to true is not appropriate here. // TODO: fix DictionaryBlock so that dictionaryIsCompacted can be set to true when the underlying block over-retains memory. return new DictionaryBlock(positionCount, dictionaryBlock, ids, false, new DictionaryId(mostSignificantBits, leastSignificantBits, sequenceId)); } }
@Override public DictionaryId apply(DictionaryBlock block) { return dictionarySourceIds.computeIfAbsent(block.getDictionarySourceId(), ignored -> randomDictionaryId()); }
public static DictionaryId randomDictionaryId() { return new DictionaryId(nodeId.getMostSignificantBits(), nodeId.getLeastSignificantBits(), sequenceGenerator.getAndIncrement()); }
/** * Create a new block from the current block by keeping the same elements * only with respect to {@code positions} that starts at {@code offset} and has length of {@code length}. * May return a view over the data in this block or may return a copy */ default Block getPositions(int[] positions, int offset, int length) { checkArrayRange(positions, offset, length); return new DictionaryBlock(offset, length, this, positions, false, randomDictionaryId()); }
@Override public Block readBlock(BlockEncodingSerde blockEncodingSerde, SliceInput sliceInput) { // positionCount int positionCount = sliceInput.readInt(); // dictionary Block dictionaryBlock = blockEncodingSerde.readBlock(sliceInput); // ids int[] ids = new int[positionCount]; sliceInput.readBytes(Slices.wrappedIntArray(ids)); // instance id long mostSignificantBits = sliceInput.readLong(); long leastSignificantBits = sliceInput.readLong(); long sequenceId = sliceInput.readLong(); // We always compact the dictionary before we send it. However, dictionaryBlock comes from sliceInput, which may over-retain memory. // As a result, setting dictionaryIsCompacted to true is not appropriate here. // TODO: fix DictionaryBlock so that dictionaryIsCompacted can be set to true when the underlying block over-retains memory. return new DictionaryBlock(positionCount, dictionaryBlock, ids, false, new DictionaryId(mostSignificantBits, leastSignificantBits, sequenceId)); } }
@Override public Block getLoadedBlock() { Block loadedDictionary = dictionary.getLoadedBlock(); if (loadedDictionary == dictionary) { return this; } return new DictionaryBlock(idsOffset, getPositionCount(), loadedDictionary, ids, false, randomDictionaryId()); }
private static DictionaryAwarePageProjection createProjection() { return new DictionaryAwarePageProjection( new TestPageProjection(), block -> randomDictionaryId()); }
DictionaryId commonSourceId = randomDictionaryId(); int commonDictionaryUsedPositions = 20; int[] commonDictionaryIds = getDictionaryIds(positionCount, commonDictionaryUsedPositions);
public DictionaryBlock(int positionCount, Block dictionary, int[] ids) { this(0, positionCount, dictionary, ids, false, randomDictionaryId()); }
public DictionaryBlock(int positionCount, Block dictionary, int[] ids, boolean dictionaryIsCompacted) { this(0, positionCount, dictionary, ids, dictionaryIsCompacted, randomDictionaryId()); }
/** * Create a new block from the current block by keeping the same elements * only with respect to {@code positions} that starts at {@code offset} and has length of {@code length}. * May return a view over the data in this block or may return a copy */ default Block getPositions(int[] positions, int offset, int length) { checkArrayRange(positions, offset, length); return new DictionaryBlock(offset, length, this, positions, false, randomDictionaryId()); }