Refine search
public void serialize(BlockBuilder out) { BlockBuilder blockBuilder = out.beginBlockEntry(); BIGINT.writeLong(blockBuilder, getCapacity()); BlockBuilder keyElements = blockBuilder.beginBlockEntry(); for (int i = 0; i < positionCount; i++) { keyType.appendTo(keyBlockBuilder, heapIndex[i], keyElements); } blockBuilder.closeEntry(); BlockBuilder valueElements = blockBuilder.beginBlockEntry(); for (int i = 0; i < positionCount; i++) { valueType.appendTo(valueBlockBuilder, heapIndex[i], valueElements); } blockBuilder.closeEntry(); out.closeEntry(); }
private static Block serializeMap(Type type, BlockBuilder builder, Object object) { Map<?, ?> map = (Map) object; if (map == null) { requireNonNull(builder, "parent builder is null").appendNull(); return null; } List<Type> typeParameters = type.getTypeParameters(); checkArgument(typeParameters.size() == 2, "map must have exactly 2 type parameter"); Type keyType = typeParameters.get(0); Type valueType = typeParameters.get(1); boolean builderSynthesized = false; if (builder == null) { builderSynthesized = true; builder = type.createBlockBuilder(new BlockBuilderStatus(), 1); } BlockBuilder currentBuilder = builder.beginBlockEntry(); for (Map.Entry<?, ?> entry : map.entrySet()) { // Hive skips map entries with null keys if (entry.getKey() != null) { serializeObject(keyType, currentBuilder, entry.getKey()); serializeObject(valueType, currentBuilder, entry.getValue()); } } builder.closeEntry(); if (builderSynthesized) { return (Block) type.getObject(builder, 0); } else { return null; } }
@Override public void start() { if (builder == null) { if (nullBuilder == null || (nullBuilder.getPositionCount() >= NULL_BUILDER_POSITIONS_THRESHOLD && nullBuilder.getSizeInBytes() >= NULL_BUILDER_SIZE_IN_BYTES_THRESHOLD)) { nullBuilder = rowType.createBlockBuilder(new BlockBuilderStatus(), NULL_BUILDER_POSITIONS_THRESHOLD); } currentEntryBuilder = nullBuilder.beginBlockEntry(); } else { while (builder.getPositionCount() < fieldIndex) { builder.appendNull(); } currentEntryBuilder = builder.beginBlockEntry(); } for (BlockConverter converter : converters) { converter.beforeValue(currentEntryBuilder); } }
@Override public void append(JsonParser parser, BlockBuilder blockBuilder) throws IOException { if (parser.getCurrentToken() == JsonToken.VALUE_NULL) { blockBuilder.appendNull(); return; } if (parser.getCurrentToken() != START_ARRAY) { throw new JsonCastException(format("Expected a json array, but got %s", parser.getText())); } BlockBuilder entryBuilder = blockBuilder.beginBlockEntry(); while (parser.nextToken() != END_ARRAY) { elementAppender.append(parser, entryBuilder); } blockBuilder.closeEntry(); } }
private static Block serializeList(Type type, BlockBuilder builder, Object object) { List<?> list = (List) object; if (list == null) { requireNonNull(builder, "parent builder is null").appendNull(); return null; } List<Type> typeParameters = type.getTypeParameters(); checkArgument(typeParameters.size() == 1, "list must have exactly 1 type parameter"); Type elementType = typeParameters.get(0); BlockBuilder currentBuilder; if (builder != null) { currentBuilder = builder.beginBlockEntry(); } else { currentBuilder = elementType.createBlockBuilder(new BlockBuilderStatus(), list.size()); } for (Object element : list) { serializeObject(elementType, currentBuilder, element); } if (builder != null) { builder.closeEntry(); return null; } else { Block resultBlock = currentBuilder.build(); return resultBlock; } }
@Override public void start() { if (builder == null) { if (nullBuilder == null || (nullBuilder.getPositionCount() >= NULL_BUILDER_POSITIONS_THRESHOLD && nullBuilder.getSizeInBytes() >= NULL_BUILDER_SIZE_IN_BYTES_THRESHOLD)) { nullBuilder = arrayType.createBlockBuilder(new BlockBuilderStatus(), NULL_BUILDER_POSITIONS_THRESHOLD); } currentEntryBuilder = nullBuilder.beginBlockEntry(); } else { while (builder.getPositionCount() < fieldIndex) { builder.appendNull(); } currentEntryBuilder = builder.beginBlockEntry(); } elementConverter.beforeValue(currentEntryBuilder); }
@Override public void append(JsonParser parser, BlockBuilder blockBuilder) throws IOException { if (parser.getCurrentToken() == JsonToken.VALUE_NULL) { blockBuilder.appendNull(); return; } if (parser.getCurrentToken() != START_ARRAY && parser.getCurrentToken() != START_OBJECT) { throw new JsonCastException(format("Expected a json array or object, but got %s", parser.getText())); } parseJsonToSingleRowBlock( parser, (SingleRowBlockWriter) blockBuilder.beginBlockEntry(), fieldAppenders, fieldNameToIndex); blockBuilder.closeEntry(); } }
private static Block serializeStruct(Type type, BlockBuilder builder, Object object, StructObjectInspector inspector) { if (object == null) { requireNonNull(builder, "parent builder is null").appendNull(); return null; } List<Type> typeParameters = type.getTypeParameters(); List<? extends StructField> allStructFieldRefs = inspector.getAllStructFieldRefs(); checkArgument(typeParameters.size() == allStructFieldRefs.size()); BlockBuilder currentBuilder; if (builder != null) { currentBuilder = builder.beginBlockEntry(); } else { currentBuilder = new InterleavedBlockBuilder(typeParameters, new BlockBuilderStatus(), typeParameters.size()); } for (int i = 0; i < typeParameters.size(); i++) { StructField field = allStructFieldRefs.get(i); serializeObject(typeParameters.get(i), currentBuilder, inspector.getStructFieldData(object, field), field.getFieldObjectInspector()); } if (builder != null) { builder.closeEntry(); return null; } else { Block resultBlock = currentBuilder.build(); return resultBlock; } }
@Override public void start() { if (builder == null) { if (nullBuilder == null || (nullBuilder.getPositionCount() >= NULL_BUILDER_POSITIONS_THRESHOLD && nullBuilder.getSizeInBytes() >= NULL_BUILDER_SIZE_IN_BYTES_THRESHOLD)) { nullBuilder = mapType.createBlockBuilder(new BlockBuilderStatus(), NULL_BUILDER_POSITIONS_THRESHOLD); } currentEntryBuilder = nullBuilder.beginBlockEntry(); } else { while (builder.getPositionCount() < fieldIndex) { builder.appendNull(); } currentEntryBuilder = builder.beginBlockEntry(); } entryConverter.beforeValue(currentEntryBuilder); }
@Override public void serialize(MinMaxNState state, BlockBuilder out) { TypedHeap heap = state.getTypedHeap(); if (heap == null) { out.appendNull(); return; } BlockBuilder blockBuilder = out.beginBlockEntry(); BIGINT.writeLong(blockBuilder, heap.getCapacity()); BlockBuilder elements = blockBuilder.beginBlockEntry(); heap.writeAll(elements); blockBuilder.closeEntry(); out.closeEntry(); }
private static RunLengthEncodedBlock createRLEBlock(Iterable<Double> percentiles, int positionCount) { BlockBuilder rleBlockBuilder = new ArrayType(DOUBLE).createBlockBuilder(new BlockBuilderStatus(), 1); BlockBuilder arrayBlockBuilder = rleBlockBuilder.beginBlockEntry(); for (double percentile : percentiles) { DOUBLE.writeDouble(arrayBlockBuilder, percentile); } rleBlockBuilder.closeEntry(); return new RunLengthEncodedBlock(rleBlockBuilder.build(), positionCount); } }
public void serialize(BlockBuilder out) { BlockBuilder mapBlockBuilder = out.beginBlockEntry(); for (int i = 0; i < keyBlockBuilder.getPositionCount(); i++) { keyType.appendTo(keyBlockBuilder, i, mapBlockBuilder); valueType.appendTo(valueBlockBuilder, i, mapBlockBuilder); } out.closeEntry(); }
public Block createLongArraysBlock(Long[] values) { ArrayType arrayType = new ArrayType(BIGINT); BlockBuilder blockBuilder = arrayType.createBlockBuilder(new BlockBuilderStatus(), values.length); for (Long value : values) { if (value == null) { blockBuilder.appendNull(); } else { BlockBuilder elementBlockBuilder = blockBuilder.beginBlockEntry(); BIGINT.writeLong(elementBlockBuilder, value); blockBuilder.closeEntry(); } } return blockBuilder.build(); }
@Override public void serialize(MultimapAggregationState state, BlockBuilder out) { if (state.isEmpty()) { out.appendNull(); return; } BlockBuilder entryBuilder = out.beginBlockEntry(); state.forEach((keyBlock, valueBlock, position) -> { BlockBuilder rowBlockBuilder = entryBuilder.beginBlockEntry(); valueType.appendTo(valueBlock, position, rowBlockBuilder); keyType.appendTo(keyBlock, position, rowBlockBuilder); entryBuilder.closeEntry(); }); out.closeEntry(); }
public static void output(ArrayType outputType, MinMaxByNState state, BlockBuilder out) { TypedKeyValueHeap heap = state.getTypedKeyValueHeap(); if (heap == null || heap.isEmpty()) { out.appendNull(); return; } Type elementType = outputType.getElementType(); BlockBuilder arrayBlockBuilder = out.beginBlockEntry(); BlockBuilder reversedBlockBuilder = elementType.createBlockBuilder(new BlockBuilderStatus(), heap.getCapacity()); long startSize = heap.getEstimatedSize(); heap.popAll(reversedBlockBuilder); state.addMemoryUsage(heap.getEstimatedSize() - startSize); for (int i = reversedBlockBuilder.getPositionCount() - 1; i >= 0; i--) { elementType.appendTo(reversedBlockBuilder, i, arrayBlockBuilder); } out.closeEntry(); }
public static Block rowBlockOf(List<Type> parameterTypes, Object... values) { RowType rowType = RowType.anonymous(parameterTypes); BlockBuilder blockBuilder = rowType.createBlockBuilder(null, 1); BlockBuilder singleRowBlockWriter = blockBuilder.beginBlockEntry(); for (int i = 0; i < values.length; i++) { appendToBlockBuilder(parameterTypes.get(i), values[i], singleRowBlockWriter); } blockBuilder.closeEntry(); return rowType.getObject(blockBuilder, 0); }
public static void output(ArrayType outputType, MinMaxNState state, BlockBuilder out) { TypedHeap heap = state.getTypedHeap(); if (heap == null || heap.isEmpty()) { out.appendNull(); return; } Type elementType = outputType.getElementType(); BlockBuilder reversedBlockBuilder = elementType.createBlockBuilder(new BlockBuilderStatus(), heap.getCapacity()); long startSize = heap.getEstimatedSize(); heap.popAll(reversedBlockBuilder); state.addMemoryUsage(heap.getEstimatedSize() - startSize); BlockBuilder arrayBlockBuilder = out.beginBlockEntry(); for (int i = reversedBlockBuilder.getPositionCount() - 1; i >= 0; i--) { elementType.appendTo(reversedBlockBuilder, i, arrayBlockBuilder); } out.closeEntry(); } }
@Override public void serialize(ArrayAggregationState state, BlockBuilder out) { if (state.isEmpty()) { out.appendNull(); } else { BlockBuilder entryBuilder = out.beginBlockEntry(); state.forEach((block, position) -> elementType.appendTo(block, position, entryBuilder)); out.closeEntry(); } }
private static Block createChannel(int positionCount, int arraySize, ArrayType arrayType) { BlockBuilder blockBuilder = arrayType.createBlockBuilder(new BlockBuilderStatus(), positionCount); for (int position = 0; position < positionCount; position++) { BlockBuilder entryBuilder = blockBuilder.beginBlockEntry(); for (int i = 0; i < arraySize; i++) { if (arrayType.getElementType().getJavaType() == long.class) { arrayType.getElementType().writeLong(entryBuilder, ThreadLocalRandom.current().nextLong()); } else if (arrayType.getElementType().equals(VARCHAR)) { arrayType.getElementType().writeSlice(entryBuilder, Slices.utf8Slice("test_string")); } else { throw new UnsupportedOperationException(); } } blockBuilder.closeEntry(); } return blockBuilder.build(); }
public static void output(Type elementType, ArrayAggregationState state, BlockBuilder out) { if (state.isEmpty()) { out.appendNull(); } else { BlockBuilder entryBuilder = out.beginBlockEntry(); state.forEach((block, position) -> elementType.appendTo(block, position, entryBuilder)); out.closeEntry(); } } }