@Override public PlanNode visitRemoteSource(RemoteSourceNode node, RewriteContext<Void> context) { return new RemoteSourceNode( node.getId(), node.getSourceFragmentIds(), canonicalizeAndDistinct(node.getOutputSymbols()), node.getOrderingScheme().map(this::canonicalizeAndDistinct), node.getExchangeType()); }
@Override public Void visitRemoteSource(RemoteSourceNode node, Integer indent) { print(indent, "- Remote%s[%s] => [%s]", node.getOrderingScheme().isPresent() ? "Merge" : "Source", Joiner.on(',').join(node.getSourceFragmentIds()), formatOutputs(node.getOutputSymbols())); printPlanNodesStatsAndCost(indent + 2, node); printStats(indent + 2, node.getId()); return null; }
public synchronized void addExchangeLocations(PlanFragmentId fragmentId, Set<RemoteTask> sourceTasks, boolean noMoreExchangeLocations) { requireNonNull(fragmentId, "fragmentId is null"); requireNonNull(sourceTasks, "sourceTasks is null"); RemoteSourceNode remoteSource = exchangeSources.get(fragmentId); checkArgument(remoteSource != null, "Unknown remote source %s. Known sources are %s", fragmentId, exchangeSources.keySet()); this.sourceTasks.putAll(remoteSource.getId(), sourceTasks); for (RemoteTask task : getAllTasks()) { ImmutableMultimap.Builder<PlanNodeId, Split> newSplits = ImmutableMultimap.builder(); for (RemoteTask sourceTask : sourceTasks) { URI exchangeLocation = sourceTask.getTaskStatus().getSelf(); newSplits.put(remoteSource.getId(), createRemoteSplitFor(task.getTaskId(), exchangeLocation)); } task.addSplits(newSplits.build()); } if (noMoreExchangeLocations) { completeSourceFragments.add(fragmentId); // is the source now complete? if (completeSourceFragments.containsAll(remoteSource.getSourceFragmentIds())) { completeSources.add(remoteSource.getId()); for (RemoteTask task : getAllTasks()) { task.noMoreSplits(remoteSource.getId()); } } } }
public RemoteSourceNode remoteSourceNode(List<PlanFragmentId> fragmentIds, List<Symbol> symbols, ExchangeNode.Type exchangeType) { return new RemoteSourceNode(idAllocator.getNextId(), fragmentIds, symbols, Optional.empty(), exchangeType); }
private static PlanFragment createJoinPlanFragment(JoinNode.Type joinType, String name, PlanFragment buildFragment, PlanFragment probeFragment) { RemoteSourceNode probe = new RemoteSourceNode(new PlanNodeId("probe_id"), probeFragment.getId(), ImmutableList.of(), Optional.empty(), REPARTITION); RemoteSourceNode build = new RemoteSourceNode(new PlanNodeId("build_id"), buildFragment.getId(), ImmutableList.of(), Optional.empty(), REPARTITION); PlanNode planNode = new JoinNode( new PlanNodeId(name + "_id"), joinType, probe, build, ImmutableList.of(), ImmutableList.<Symbol>builder() .addAll(probe.getOutputSymbols()) .addAll(build.getOutputSymbols()) .build(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.empty()); return createFragment(planNode); }
@Override public PlanNode visitRemoteSource(RemoteSourceNode node, RewriteContext<Void> context) { return new RemoteSourceNode(node.getId(), node.getSourceFragmentIds(), canonicalizeAndDistinct(node.getOutputSymbols())); }
@Override public Void visitRemoteSource(RemoteSourceNode node, Void context) { node.getSourceFragmentIds() .forEach(this::processFragment); return null; }
@Override public Void visitRemoteSource(RemoteSourceNode node, Integer indent) { print(indent, "- RemoteSource[%s] => [%s]", Joiner.on(',').join(node.getSourceFragmentIds()), formatOutputs(node.getOutputSymbols())); return null; }
private PhysicalOperation createMergeSource(RemoteSourceNode node, LocalExecutionPlanContext context) { checkArgument(node.getOrderingScheme().isPresent(), "orderingScheme is absent"); // merging remote source must have a single driver context.setDriverInstanceCount(1); OrderingScheme orderingScheme = node.getOrderingScheme().get(); ImmutableMap<Symbol, Integer> layout = makeLayout(node); List<Integer> sortChannels = getChannelsForSymbols(orderingScheme.getOrderBy(), layout); List<SortOrder> sortOrder = orderingScheme.getOrderingList(); List<Type> types = getSourceOperatorTypes(node, context.getTypes()); ImmutableList<Integer> outputChannels = IntStream.range(0, types.size()) .boxed() .collect(toImmutableList()); OperatorFactory operatorFactory = new MergeOperatorFactory( context.getNextOperatorId(), node.getId(), exchangeClientSupplier, new PagesSerdeFactory(blockEncodingSerde, isExchangeCompressionEnabled(session)), orderingCompiler, types, outputChannels, sortChannels, sortOrder); return new PhysicalOperation(operatorFactory, makeLayout(node), context, UNGROUPED_EXECUTION); }
@Override public Void visitRemoteSource(RemoteSourceNode node, Void context) { builder.addAll(node.getOutputSymbols()); return null; }
private PhysicalOperation createRemoteSource(RemoteSourceNode node, LocalExecutionPlanContext context) { if (!context.getDriverInstanceCount().isPresent()) { context.setDriverInstanceCount(getTaskConcurrency(session)); } OperatorFactory operatorFactory = new ExchangeOperatorFactory( context.getNextOperatorId(), node.getId(), exchangeClientSupplier, new PagesSerdeFactory(blockEncodingSerde, isExchangeCompressionEnabled(session))); return new PhysicalOperation(operatorFactory, makeLayout(node), context, UNGROUPED_EXECUTION); }
@Override public Void visitRemoteSource(RemoteSourceNode node, Void context) { printNode(node, (node.getOrderingScheme().isPresent() ? "Merge" : "Exchange") + " 1:N", NODE_COLORS.get(NodeType.EXCHANGE)); return null; }
private static PlanFragment createExchangePlanFragment(String name, PlanFragment... fragments) { PlanNode planNode = new RemoteSourceNode( new PlanNodeId(name + "_id"), Stream.of(fragments) .map(PlanFragment::getId) .collect(toImmutableList()), fragments[0].getPartitioningScheme().getOutputLayout(), Optional.empty(), REPARTITION); return createFragment(planNode); }
private static PlanFragment createBroadcastJoinPlanFragment(String name, PlanFragment buildFragment) { Symbol symbol = new Symbol("column"); PlanNode tableScan = new TableScanNode( new PlanNodeId(name), new TableHandle(new ConnectorId("test"), new TestingTableHandle()), ImmutableList.of(symbol), ImmutableMap.of(symbol, new TestingColumnHandle("column"))); RemoteSourceNode remote = new RemoteSourceNode(new PlanNodeId("build_id"), buildFragment.getId(), ImmutableList.of(), Optional.empty(), REPLICATE); PlanNode join = new JoinNode( new PlanNodeId(name + "_id"), INNER, tableScan, remote, ImmutableList.of(), ImmutableList.<Symbol>builder() .addAll(tableScan.getOutputSymbols()) .addAll(remote.getOutputSymbols()) .build(), Optional.empty(), Optional.empty(), Optional.empty(), Optional.of(REPLICATED)); return createFragment(join); }
private SqlStageExecution(StageStateMachine stateMachine, RemoteTaskFactory remoteTaskFactory, NodeTaskMap nodeTaskMap, boolean summarizeTaskInfo, Executor executor, FailureDetector failureDetector) { this.stateMachine = stateMachine; this.remoteTaskFactory = requireNonNull(remoteTaskFactory, "remoteTaskFactory is null"); this.nodeTaskMap = requireNonNull(nodeTaskMap, "nodeTaskMap is null"); this.summarizeTaskInfo = summarizeTaskInfo; this.executor = requireNonNull(executor, "executor is null"); this.failureDetector = requireNonNull(failureDetector, "failureDetector is null"); ImmutableMap.Builder<PlanFragmentId, RemoteSourceNode> fragmentToExchangeSource = ImmutableMap.builder(); for (RemoteSourceNode remoteSourceNode : stateMachine.getFragment().getRemoteSourceNodes()) { for (PlanFragmentId planFragmentId : remoteSourceNode.getSourceFragmentIds()) { fragmentToExchangeSource.put(planFragmentId, remoteSourceNode); } } this.exchangeSources = fragmentToExchangeSource.build(); }
public synchronized void addExchangeLocations(PlanFragmentId fragmentId, Set<URI> exchangeLocations, boolean noMoreExchangeLocations) { requireNonNull(fragmentId, "fragmentId is null"); requireNonNull(exchangeLocations, "exchangeLocations is null"); RemoteSourceNode remoteSource = exchangeSources.get(fragmentId); checkArgument(remoteSource != null, "Unknown remote source %s. Known sources are %s", fragmentId, exchangeSources.keySet()); this.exchangeLocations.putAll(remoteSource.getId(), exchangeLocations); for (RemoteTask task : getAllTasks()) { ImmutableMultimap.Builder<PlanNodeId, Split> newSplits = ImmutableMultimap.builder(); for (URI exchangeLocation : exchangeLocations) { newSplits.put(remoteSource.getId(), createRemoteSplitFor(task.getTaskInfo().getTaskId(), exchangeLocation)); } task.addSplits(newSplits.build()); } if (noMoreExchangeLocations) { completeSourceFragments.add(fragmentId); // is the source now complete? if (completeSourceFragments.containsAll(remoteSource.getSourceFragmentIds())) { completeSources.add(remoteSource.getId()); for (RemoteTask task : getAllTasks()) { task.noMoreSplits(remoteSource.getId()); } } } }
@Override public PhysicalOperation visitRemoteSource(RemoteSourceNode node, LocalExecutionPlanContext context) { List<Type> types = getSourceOperatorTypes(node, context.getTypes()); OperatorFactory operatorFactory = new ExchangeOperatorFactory(context.getNextOperatorId(), node.getId(), exchangeClientSupplier, types); return new PhysicalOperation(operatorFactory, makeLayout(node)); }
@Override public PhysicalOperation visitRemoteSource(RemoteSourceNode node, LocalExecutionPlanContext context) { if (node.getOrderingScheme().isPresent()) { return createMergeSource(node, context); } return createRemoteSource(node, context); }
@Override public PlanNode visitExchange(ExchangeNode exchange, RewriteContext<FragmentProperties> context) { if (exchange.getScope() != REMOTE) { return context.defaultRewrite(exchange, context.get()); } PartitioningScheme partitioningScheme = exchange.getPartitioningScheme(); if (exchange.getType() == ExchangeNode.Type.GATHER) { context.get().setSingleNodeDistribution(); } else if (exchange.getType() == ExchangeNode.Type.REPARTITION) { context.get().setDistribution(partitioningScheme.getPartitioning().getHandle(), metadata, session); } ImmutableList.Builder<SubPlan> builder = ImmutableList.builder(); for (int sourceIndex = 0; sourceIndex < exchange.getSources().size(); sourceIndex++) { FragmentProperties childProperties = new FragmentProperties(partitioningScheme.translateOutputLayout(exchange.getInputs().get(sourceIndex))); builder.add(buildSubPlan(exchange.getSources().get(sourceIndex), childProperties, context)); } List<SubPlan> children = builder.build(); context.get().addChildren(children); List<PlanFragmentId> childrenIds = children.stream() .map(SubPlan::getFragment) .map(PlanFragment::getId) .collect(toImmutableList()); return new RemoteSourceNode(exchange.getId(), childrenIds, exchange.getOutputSymbols(), exchange.getOrderingScheme(), exchange.getType()); }
ImmutableMap.of(symbol, new TestingColumnHandle("column"))); RemoteSourceNode remote = new RemoteSourceNode(new PlanNodeId("remote_id"), new PlanFragmentId("plan_fragment_id"), ImmutableList.of(), Optional.empty(), GATHER); PlanFragment testFragment = new PlanFragment( new PlanFragmentId("plan_id"), ImmutableList.<Symbol>builder() .addAll(tableScan.getOutputSymbols()) .addAll(remote.getOutputSymbols()) .build(), Optional.empty(),