@VisibleForTesting static Comparator<ActualProperties> streamingExecutionPreference(PreferredProperties preferred) { // Calculating the matches can be a bit expensive, so cache the results between comparisons LoadingCache<List<LocalProperty<Symbol>>, List<Optional<LocalProperty<Symbol>>>> matchCache = CacheBuilder.newBuilder() .build(CacheLoader.from(actualProperties -> LocalProperties.match(actualProperties, preferred.getLocalProperties()))); return (actual1, actual2) -> { List<Optional<LocalProperty<Symbol>>> matchLayout1 = matchCache.getUnchecked(actual1.getLocalProperties()); List<Optional<LocalProperty<Symbol>>> matchLayout2 = matchCache.getUnchecked(actual2.getLocalProperties()); return ComparisonChain.start() .compareTrueFirst(hasLocalOptimization(preferred.getLocalProperties(), matchLayout1), hasLocalOptimization(preferred.getLocalProperties(), matchLayout2)) .compareTrueFirst(meetsPartitioningRequirements(preferred, actual1), meetsPartitioningRequirements(preferred, actual2)) .compare(matchLayout1, matchLayout2, matchedLayoutPreference()) .result(); }; }
public PreferredProperties mergeWithParent(PreferredProperties parent) { List<LocalProperty<Symbol>> newLocal = ImmutableList.<LocalProperty<Symbol>>builder() .addAll(localProperties) .addAll(parent.getLocalProperties()) .build(); Builder builder = builder() .local(newLocal); if (globalProperties.isPresent()) { Global currentGlobal = globalProperties.get(); Global newGlobal = parent.getGlobalProperties() .map(currentGlobal::mergeWithParent) .orElse(currentGlobal); builder.global(newGlobal); } else { builder.global(parent.getGlobalProperties()); } return builder.build(); }
@Override public PlanWithProperties visitIndexJoin(IndexJoinNode node, PreferredProperties preferredProperties) { List<Symbol> joinColumns = node.getCriteria().stream() .map(IndexJoinNode.EquiJoinClause::getProbe) .collect(toImmutableList()); // Only prefer grouping on join columns if no parent local property preferences List<LocalProperty<Symbol>> desiredLocalProperties = preferredProperties.getLocalProperties().isEmpty() ? grouped(joinColumns) : ImmutableList.of(); PlanWithProperties probeSource = node.getProbeSource().accept(this, PreferredProperties.partitionedWithLocal(ImmutableSet.copyOf(joinColumns), desiredLocalProperties) .mergeWithParent(preferredProperties)); ActualProperties probeProperties = probeSource.getProperties(); PlanWithProperties indexSource = node.getIndexSource().accept(this, PreferredProperties.any()); // TODO: allow repartitioning if unpartitioned to increase parallelism if (shouldRepartitionForIndexJoin(joinColumns, preferredProperties, probeProperties)) { probeSource = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), REMOTE, probeSource.getNode(), joinColumns, node.getProbeHashSymbol()), probeProperties); } // TODO: if input is grouped, create streaming join // index side is really a nested-loops plan, so don't add exchanges PlanNode result = ChildReplacer.replaceChildren(node, ImmutableList.of(probeSource.getNode(), node.getIndexSource())); return new PlanWithProperties(result, deriveProperties(result, ImmutableList.of(probeSource.getProperties(), indexSource.getProperties()))); }
@Override public List<Optional<LocalProperty<Symbol>>> load(List<LocalProperty<Symbol>> actualProperties) { return LocalProperties.match(actualProperties, preferred.getLocalProperties()); } });
@VisibleForTesting static Comparator<ActualProperties> streamingExecutionPreference(PreferredProperties preferred) { // Calculating the matches can be a bit expensive, so cache the results between comparisons LoadingCache<List<LocalProperty<Symbol>>, List<Optional<LocalProperty<Symbol>>>> matchCache = CacheBuilder.newBuilder() .build(new CacheLoader<List<LocalProperty<Symbol>>, List<Optional<LocalProperty<Symbol>>>>() { @Override public List<Optional<LocalProperty<Symbol>>> load(List<LocalProperty<Symbol>> actualProperties) { return LocalProperties.match(actualProperties, preferred.getLocalProperties()); } }); return (actual1, actual2) -> { List<Optional<LocalProperty<Symbol>>> matchLayout1 = matchCache.getUnchecked(actual1.getLocalProperties()); List<Optional<LocalProperty<Symbol>>> matchLayout2 = matchCache.getUnchecked(actual2.getLocalProperties()); return ComparisonChain.start() .compareTrueFirst(hasLocalOptimization(preferred.getLocalProperties(), matchLayout1), hasLocalOptimization(preferred.getLocalProperties(), matchLayout2)) .compareTrueFirst(meetsPartitioningRequirements(preferred, actual1), meetsPartitioningRequirements(preferred, actual2)) .compare(matchLayout1, matchLayout2, matchedLayoutPreference()) .result(); }; }
.addAll(parentProperties.getLocalProperties()) .build();
@Override public PlanWithProperties visitIndexJoin(IndexJoinNode node, Context context) { List<Symbol> joinColumns = Lists.transform(node.getCriteria(), IndexJoinNode.EquiJoinClause::getProbe); // Only prefer grouping on join columns if no parent local property preferences List<LocalProperty<Symbol>> desiredLocalProperties = context.getPreferredProperties().getLocalProperties().isEmpty() ? grouped(joinColumns) : ImmutableList.of(); PlanWithProperties probeSource = node.getProbeSource().accept(this, context.withPreferredProperties(PreferredProperties.derivePreferences(context.getPreferredProperties(), ImmutableSet.copyOf(joinColumns), desiredLocalProperties))); ActualProperties probeProperties = probeSource.getProperties(); PlanWithProperties indexSource = node.getIndexSource().accept(this, context.withPreferredProperties(PreferredProperties.any())); // TODO: allow repartitioning if unpartitioned to increase parallelism if (shouldRepartitionForIndexJoin(joinColumns, context.getPreferredProperties(), probeProperties)) { probeSource = withDerivedProperties( partitionedExchange(idAllocator.getNextId(), probeSource.getNode(), joinColumns, node.getProbeHashSymbol()), probeProperties); } // TODO: if input is grouped, create streaming join // index side is really a nested-loops plan, so don't add exchanges PlanNode result = ChildReplacer.replaceChildren(node, ImmutableList.of(probeSource.getNode(), node.getIndexSource())); return new PlanWithProperties(result, deriveProperties(result, ImmutableList.of(probeSource.getProperties(), indexSource.getProperties()))); }