@Override public Block apply(Block block) { if (elementCoercer == null) { return block; } ColumnarArray arrayBlock = toColumnarArray(block); Block elementsBlock = elementCoercer.apply(arrayBlock.getElementsBlock()); boolean[] valueIsNull = new boolean[arrayBlock.getPositionCount()]; int[] offsets = new int[arrayBlock.getPositionCount() + 1]; for (int i = 0; i < arrayBlock.getPositionCount(); i++) { valueIsNull[i] = arrayBlock.isNull(i); offsets[i + 1] = offsets[i] + arrayBlock.getLength(i); } return ArrayBlock.fromElementBlock(arrayBlock.getPositionCount(), Optional.of(valueIsNull), offsets, elementsBlock); } }
private static ColumnarArray toColumnarArray(DictionaryBlock dictionaryBlock) { ColumnarArray columnarArray = toColumnarArray(dictionaryBlock.getDictionary()); // build new offsets int[] offsets = new int[dictionaryBlock.getPositionCount() + 1]; for (int position = 0; position < dictionaryBlock.getPositionCount(); position++) { int dictionaryId = dictionaryBlock.getId(position); offsets[position + 1] = offsets[position] + columnarArray.getLength(dictionaryId); } // reindex dictionary int[] dictionaryIds = new int[offsets[dictionaryBlock.getPositionCount()]]; int nextDictionaryIndex = 0; for (int position = 0; position < dictionaryBlock.getPositionCount(); position++) { int dictionaryId = dictionaryBlock.getId(position); int length = columnarArray.getLength(dictionaryId); // adjust to the element block start offset int startOffset = columnarArray.getOffset(dictionaryId) - columnarArray.getOffset(0); for (int entryIndex = 0; entryIndex < length; entryIndex++) { dictionaryIds[nextDictionaryIndex] = startOffset + entryIndex; nextDictionaryIndex++; } } return new ColumnarArray( dictionaryBlock, 0, offsets, new DictionaryBlock(dictionaryIds.length, columnarArray.getElementsBlock(), dictionaryIds)); }
public static ColumnarArray toColumnarArray(Block block) { requireNonNull(block, "block is null"); if (block instanceof DictionaryBlock) { return toColumnarArray((DictionaryBlock) block); } if (block instanceof RunLengthEncodedBlock) { return toColumnarArray((RunLengthEncodedBlock) block); } if (!(block instanceof AbstractArrayBlock)) { throw new IllegalArgumentException("Invalid array block: " + block.getClass().getName()); } AbstractArrayBlock arrayBlock = (AbstractArrayBlock) block; Block elementsBlock = arrayBlock.getRawElementBlock(); // trim elements to just visible region int elementsOffset = 0; int elementsLength = 0; if (arrayBlock.getPositionCount() > 0) { elementsOffset = arrayBlock.getOffset(0); elementsLength = arrayBlock.getOffset(arrayBlock.getPositionCount()) - elementsOffset; } elementsBlock = elementsBlock.getRegion(elementsOffset, elementsLength); return new ColumnarArray(block, arrayBlock.getOffsetBase(), arrayBlock.getOffsets(), elementsBlock); }
fieldExtractor = block -> ImmutableList.of(toColumnarArray(block).getElementsBlock()); fieldBuilders = ImmutableList.of(new ColumnStatisticsValidation(Iterables.getOnlyElement(type.getTypeParameters())));
@Override public void writeBlock(Block block) { checkState(!closed); checkArgument(block.getPositionCount() > 0, "Block is empty"); ColumnarArray columnarArray = toColumnarArray(block); writeColumnarArray(columnarArray); }
fieldExtractor = block -> ImmutableList.of(toColumnarArray(block).getElementsBlock()); fieldBuilders = ImmutableList.of(new ColumnStatisticsValidation(Iterables.getOnlyElement(type.getTypeParameters())));
public static ColumnarArray toColumnarArray(Block block) { requireNonNull(block, "block is null"); if (block instanceof DictionaryBlock) { return toColumnarArray((DictionaryBlock) block); } if (block instanceof RunLengthEncodedBlock) { return toColumnarArray((RunLengthEncodedBlock) block); } if (!(block instanceof AbstractArrayBlock)) { throw new IllegalArgumentException("Invalid array block: " + block.getClass().getName()); } AbstractArrayBlock arrayBlock = (AbstractArrayBlock) block; Block elementsBlock = arrayBlock.getRawElementBlock(); // trim elements to just visible region int elementsOffset = 0; int elementsLength = 0; if (arrayBlock.getPositionCount() > 0) { elementsOffset = arrayBlock.getOffset(0); elementsLength = arrayBlock.getOffset(arrayBlock.getPositionCount()) - elementsOffset; } elementsBlock = elementsBlock.getRegion(elementsOffset, elementsLength); return new ColumnarArray(block, arrayBlock.getOffsetBase(), arrayBlock.getOffsets(), elementsBlock); }
@Override public void writeBlock(Block block) { checkState(!closed); checkArgument(block.getPositionCount() > 0, "Block is empty"); ColumnarArray columnarArray = toColumnarArray(block); writeColumnarArray(columnarArray); }
private void writeColumnarArray(ColumnarArray columnarArray) { // write nulls and lengths for (int position = 0; position < columnarArray.getPositionCount(); position++) { boolean present = !columnarArray.isNull(position); presentStream.writeBoolean(present); if (present) { nonNullValueCount++; lengthStream.writeLong(columnarArray.getLength(position)); } } // write element values Block elementsBlock = columnarArray.getElementsBlock(); if (elementsBlock.getPositionCount() > 0) { elementWriter.writeBlock(elementsBlock); } }
private static ColumnarArray toColumnarArray(RunLengthEncodedBlock rleBlock) { ColumnarArray columnarArray = toColumnarArray(rleBlock.getValue()); // build new offsets block int[] offsets = new int[rleBlock.getPositionCount() + 1]; int valueLength = columnarArray.getLength(0); for (int i = 0; i < offsets.length; i++) { offsets[i] = i * valueLength; } // create indexes for a dictionary block of the elements int[] dictionaryIds = new int[rleBlock.getPositionCount() * valueLength]; int nextDictionaryIndex = 0; for (int position = 0; position < rleBlock.getPositionCount(); position++) { for (int entryIndex = 0; entryIndex < valueLength; entryIndex++) { dictionaryIds[nextDictionaryIndex] = entryIndex; nextDictionaryIndex++; } } return new ColumnarArray( rleBlock, 0, offsets, new DictionaryBlock(dictionaryIds.length, columnarArray.getElementsBlock(), dictionaryIds)); }
private static <T> void assertColumnarArray(Block block, T[] expectedValues) { ColumnarArray columnarArray = toColumnarArray(block); assertEquals(columnarArray.getPositionCount(), expectedValues.length); Block elementsBlock = columnarArray.getElementsBlock(); int elementsPosition = 0; for (int position = 0; position < expectedValues.length; position++) { T expectedArray = expectedValues[position]; assertEquals(columnarArray.isNull(position), expectedArray == null); assertEquals(columnarArray.getLength(position), expectedArray == null ? 0 : Array.getLength(expectedArray)); for (int i = 0; i < columnarArray.getLength(position); i++) { Object expectedElement = Array.get(expectedArray, i); assertBlockPosition(elementsBlock, elementsPosition, expectedElement); elementsPosition++; } } }
private static ColumnarArray toColumnarArray(DictionaryBlock dictionaryBlock) { ColumnarArray columnarArray = toColumnarArray(dictionaryBlock.getDictionary()); // build new offsets int[] offsets = new int[dictionaryBlock.getPositionCount() + 1]; for (int position = 0; position < dictionaryBlock.getPositionCount(); position++) { int dictionaryId = dictionaryBlock.getId(position); offsets[position + 1] = offsets[position] + columnarArray.getLength(dictionaryId); } // reindex dictionary int[] dictionaryIds = new int[offsets[dictionaryBlock.getPositionCount()]]; int nextDictionaryIndex = 0; for (int position = 0; position < dictionaryBlock.getPositionCount(); position++) { int dictionaryId = dictionaryBlock.getId(position); int length = columnarArray.getLength(dictionaryId); // adjust to the element block start offset int startOffset = columnarArray.getOffset(dictionaryId) - columnarArray.getOffset(0); for (int entryIndex = 0; entryIndex < length; entryIndex++) { dictionaryIds[nextDictionaryIndex] = startOffset + entryIndex; nextDictionaryIndex++; } } return new ColumnarArray( dictionaryBlock, 0, offsets, new DictionaryBlock(dictionaryIds.length, columnarArray.getElementsBlock(), dictionaryIds)); }
private void writeColumnarArray(ColumnarArray columnarArray) { // write nulls and lengths for (int position = 0; position < columnarArray.getPositionCount(); position++) { boolean present = !columnarArray.isNull(position); presentStream.writeBoolean(present); if (present) { nonNullValueCount++; lengthStream.writeLong(columnarArray.getLength(position)); } } // write element values Block elementsBlock = columnarArray.getElementsBlock(); if (elementsBlock.getPositionCount() > 0) { elementWriter.writeBlock(elementsBlock); } }
private static ColumnarArray toColumnarArray(RunLengthEncodedBlock rleBlock) { ColumnarArray columnarArray = toColumnarArray(rleBlock.getValue()); // build new offsets block int[] offsets = new int[rleBlock.getPositionCount() + 1]; int valueLength = columnarArray.getLength(0); for (int i = 0; i < offsets.length; i++) { offsets[i] = i * valueLength; } // create indexes for a dictionary block of the elements int[] dictionaryIds = new int[rleBlock.getPositionCount() * valueLength]; int nextDictionaryIndex = 0; for (int position = 0; position < rleBlock.getPositionCount(); position++) { for (int entryIndex = 0; entryIndex < valueLength; entryIndex++) { dictionaryIds[nextDictionaryIndex] = entryIndex; nextDictionaryIndex++; } } return new ColumnarArray( rleBlock, 0, offsets, new DictionaryBlock(dictionaryIds.length, columnarArray.getElementsBlock(), dictionaryIds)); }