private static int getSize(NullableValue nullableValue) { if (nullableValue.isNull()) { return 0; } Object value = nullableValue.getValue(); checkArgument(value instanceof Slice, "value is expected to be of Slice type"); return ((Slice) value).length(); }
@VisibleForTesting static long calculateDistinctPartitionKeys( HiveColumnHandle column, List<HivePartition> partitions, Map<String, PartitionStatistics> statistics, double averageRowsPerPartition) { return partitions.stream() // consider only non empty partitions .filter(partition -> getPartitionRowCount(partition.getPartitionId(), statistics).orElse(averageRowsPerPartition) > 0) .map(partition -> partition.getKeys().get(column)) .filter(value -> !value.isNull()) .distinct() .count(); }
@VisibleForTesting static Optional<DoubleRange> calculateRangeForPartitioningKey(HiveColumnHandle column, Type type, List<HivePartition> partitions) { if (!isRangeSupported(type)) { return Optional.empty(); } List<Double> values = partitions.stream() .map(HivePartition::getKeys) .map(keys -> keys.get(column)) .filter(value -> !value.isNull()) .map(NullableValue::getValue) .map(value -> convertPartitionValueToDouble(type, value)) .collect(toImmutableList()); if (values.isEmpty()) { return Optional.empty(); } double min = values.get(0); double max = values.get(0); for (Double value : values) { if (value > max) { max = value; } if (value < min) { min = value; } } return Optional.of(new DoubleRange(min, max)); }
@VisibleForTesting static double calculateNullsFractionForPartitioningKey( HiveColumnHandle column, List<HivePartition> partitions, Map<String, PartitionStatistics> statistics, double averageRowsPerPartition, double rowCount) { if (rowCount == 0) { return 0; } double estimatedNullsCount = partitions.stream() .filter(partition -> partition.getKeys().get(column).isNull()) .map(HivePartition::getPartitionId) .mapToDouble(partitionName -> getPartitionRowCount(partitionName, statistics).orElse(averageRowsPerPartition)) .sum(); return normalizeFraction(estimatedNullsCount / rowCount); }
private Optional<Expression> coerceComparisonWithRounding( Type symbolExpressionType, Expression symbolExpression, NullableValue nullableValue, ComparisonExpression.Operator comparisonOperator) { requireNonNull(nullableValue, "nullableValue is null"); if (nullableValue.isNull()) { return Optional.empty(); } Type valueType = nullableValue.getType(); Object value = nullableValue.getValue(); return floorValue(valueType, symbolExpressionType, value) .map((floorValue) -> rewriteComparisonExpression(symbolExpressionType, symbolExpression, valueType, value, floorValue, comparisonOperator)); }
@Override public ActualProperties visitTableScan(TableScanNode node, List<ActualProperties> inputProperties) { checkArgument(node.getLayout().isPresent(), "table layout has not yet been chosen"); TableLayout layout = metadata.getLayout(session, node.getLayout().get()); Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse(); ActualProperties.Builder properties = ActualProperties.builder(); // Globally constant assignments Map<ColumnHandle, NullableValue> globalConstants = new HashMap<>(); extractFixedValues(node.getCurrentConstraint()).orElse(ImmutableMap.of()) .entrySet().stream() .filter(entry -> !entry.getValue().isNull()) .forEach(entry -> globalConstants.put(entry.getKey(), entry.getValue())); Map<Symbol, NullableValue> symbolConstants = globalConstants.entrySet().stream() .filter(entry -> assignments.containsKey(entry.getKey())) .collect(toMap(entry -> assignments.get(entry.getKey()), Map.Entry::getValue)); properties.constants(symbolConstants); // Partitioning properties properties.global(deriveGlobalProperties(layout, assignments, globalConstants)); // Append the global constants onto the local properties to maximize their translation potential List<LocalProperty<ColumnHandle>> constantAppendedLocalProperties = ImmutableList.<LocalProperty<ColumnHandle>>builder() .addAll(globalConstants.keySet().stream().map(ConstantProperty::new).iterator()) .addAll(layout.getLocalProperties()) .build(); properties.local(LocalProperties.translate(constantAppendedLocalProperties, column -> Optional.ofNullable(assignments.get(column)))); return properties.build(); }
@Override public StreamProperties visitTableScan(TableScanNode node, List<StreamProperties> inputProperties) { checkArgument(node.getLayout().isPresent(), "table layout has not yet been chosen"); TableLayout layout = metadata.getLayout(session, node.getLayout().get()); Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse(); // Globally constant assignments Set<ColumnHandle> constants = new HashSet<>(); extractFixedValues(node.getCurrentConstraint()).orElse(ImmutableMap.of()) .entrySet().stream() .filter(entry -> !entry.getValue().isNull()) // TODO consider allowing nulls .forEach(entry -> constants.add(entry.getKey())); Optional<Set<Symbol>> streamPartitionSymbols = layout.getStreamPartitioningColumns() .flatMap(columns -> getNonConstantSymbols(columns, assignments, constants)); // if we are partitioned on empty set, we must say multiple of unknown partitioning, because // the connector does not guarantee a single split in this case (since it might not understand // that the value is a constant). if (streamPartitionSymbols.isPresent() && streamPartitionSymbols.get().isEmpty()) { return new StreamProperties(MULTIPLE, Optional.empty(), false); } return new StreamProperties(MULTIPLE, streamPartitionSymbols, false); }
for (Entry<ColumnHandle, NullableValue> entry : bindings.entrySet()) { HiveColumnHandle colHandle = (HiveColumnHandle) entry.getKey(); if (!entry.getValue().isNull() && bucketColumns.contains(colHandle.getName())) { bucketBindings.put(colHandle.getName(), entry.getValue().getValue());
.entrySet().stream() .filter(entry -> !indexableColumns.contains(entry.getKey())) .filter(entry -> !entry.getValue().isNull()) // strip nulls since meaningless in index join lookups .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
private static Domain buildColumnDomain(ColumnHandle column, List<HivePartition> partitions) { checkArgument(!partitions.isEmpty(), "partitions cannot be empty"); boolean hasNull = false; List<Object> nonNullValues = new ArrayList<>(); Type type = null; for (HivePartition partition : partitions) { NullableValue value = partition.getKeys().get(column); if (value == null) { throw new PrestoException(HIVE_UNKNOWN_ERROR, format("Partition %s does not have a value for partition column %s", partition, column)); } if (value.isNull()) { hasNull = true; } else { nonNullValues.add(value.getValue()); } if (type == null) { type = value.getType(); } } if (!nonNullValues.isEmpty()) { Domain domain = Domain.multipleValues(type, nonNullValues); if (hasNull) { return domain.union(Domain.onlyNull(type)); } return domain; } return Domain.onlyNull(type); }
static String getStringValue(NullableValue value) { if ((value == null) || value.isNull()) { return null; } return ((Slice) value.getValue()).toStringUtf8(); } }
@Override public ActualProperties visitTableScan(TableScanNode node, List<ActualProperties> inputProperties) { checkArgument(node.getLayout().isPresent(), "table layout has not yet been chosen"); TableLayout layout = metadata.getLayout(session, node.getLayout().get()); Map<ColumnHandle, Symbol> assignments = ImmutableBiMap.copyOf(node.getAssignments()).inverse(); ActualProperties.Builder properties = ActualProperties.builder(); // Globally constant assignments Map<ColumnHandle, NullableValue> globalConstants = new HashMap<>(); extractFixedValues(node.getCurrentConstraint()).orElse(ImmutableMap.of()) .entrySet().stream() .filter(entry -> !entry.getValue().isNull()) .forEach(entry -> globalConstants.put(entry.getKey(), entry.getValue())); Map<Symbol, NullableValue> symbolConstants = globalConstants.entrySet().stream() .filter(entry -> assignments.containsKey(entry.getKey())) .collect(toMap(entry -> assignments.get(entry.getKey()), Map.Entry::getValue)); properties.constants(symbolConstants); // Partitioning properties properties.global(deriveGlobalProperties(layout, assignments, globalConstants)); // Append the global constants onto the local properties to maximize their translation potential List<LocalProperty<ColumnHandle>> constantAppendedLocalProperties = ImmutableList.<LocalProperty<ColumnHandle>>builder() .addAll(globalConstants.keySet().stream().map(column -> new ConstantProperty<>(column)).iterator()) .addAll(layout.getLocalProperties()) .build(); properties.local(LocalProperties.translate(constantAppendedLocalProperties, column -> Optional.ofNullable(assignments.get(column)))); return properties.build(); }
.entrySet().stream() .filter(entry -> !indexableColumns.contains(entry.getKey())) .filter(entry -> !entry.getValue().isNull()) // strip nulls since meaningless in index join lookups .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
for (Entry<ColumnHandle, NullableValue> entry : bindings.entrySet()) { HiveColumnHandle colHandle = (HiveColumnHandle) entry.getKey(); if (!entry.getValue().isNull() && bucketColumns.contains(colHandle.getName())) { bucketBindings.put(colHandle.getName(), entry.getValue().getValue());
private Optional<NullableValue> coerce(NullableValue value, Type targetType) { if (!TypeRegistry.canCoerce(value.getType(), targetType)) { return Optional.empty(); } if (value.isNull()) { return Optional.of(NullableValue.asNull(targetType)); } Object coercedValue = new FunctionInvoker(metadata.getFunctionRegistry()) .invoke(metadata.getFunctionRegistry().getCoercion(value.getType(), targetType), session.toConnectorSession(), value.getValue()); return Optional.of(NullableValue.of(targetType, coercedValue)); }
private static Expression coerceDoubleToLongComparison(NormalizedSimpleComparison normalized) checkArgument(!normalized.getValue().isNull(), "Value should not be null"); QualifiedNameReference reference = normalized.getNameReference(); Double value = (Double) normalized.getValue().getValue();