/** * Constructs a new sink candidate node that uses <i>NONE</i> as its local strategy. Note that * local sorting and range partitioning are handled by the incoming channel already. * * @param template The template optimizer node that this candidate is created for. */ public SinkPlanNode(DataSinkNode template, String nodeName, Channel input) { super(template, nodeName, input, DriverStrategy.NONE); this.globalProps = input.getGlobalProperties().clone(); this.localProps = input.getLocalProperties().clone(); }
@Override public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) { boolean[] inputOrders = in2.getLocalProperties().getOrdering() == null ? null : in2.getLocalProperties().getOrdering().getFieldSortDirections(); if (inputOrders == null || inputOrders.length < this.keys2.size()) { throw new CompilerException("BUG: The input strategy does not sufficiently describe the sort orders for a CoGroup operator."); } else if (inputOrders.length > this.keys2.size()) { boolean[] tmp = new boolean[this.keys2.size()]; System.arraycopy(inputOrders, 0, tmp, 0, tmp.length); inputOrders = tmp; } return new DualInputPlanNode(node, "CoGroup ("+node.getOperator().getName()+")", in1, in2, DriverStrategy.CO_GROUP, this.keys1, this.keys2, inputOrders); }
@Override public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) { boolean[] inputOrders = in1.getLocalProperties().getOrdering() == null ? null : in1.getLocalProperties().getOrdering().getFieldSortDirections(); if (inputOrders == null || inputOrders.length < this.keys1.size()) { throw new CompilerException("BUG: The input strategy does not sufficiently describe the sort orders for a CoGroup operator."); } else if (inputOrders.length > this.keys1.size()) { boolean[] tmp = new boolean[this.keys1.size()]; System.arraycopy(inputOrders, 0, tmp, 0, tmp.length); inputOrders = tmp; } return new DualInputPlanNode(node, "CoGroup ("+node.getOperator().getName()+")", in1, in2, DriverStrategy.CO_GROUP, this.keys1, this.keys2, inputOrders); }
@Override public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) { boolean[] inputOrders = in1.getLocalProperties().getOrdering() == null ? null : in1.getLocalProperties().getOrdering().getFieldSortDirections(); if (inputOrders == null || inputOrders.length < this.keys1.size()) { throw new CompilerException("BUG: The input strategy does not sufficiently describe the sort orders for a CoGroup operator."); } else if (inputOrders.length > this.keys1.size()) { boolean[] tmp = new boolean[this.keys1.size()]; System.arraycopy(inputOrders, 0, tmp, 0, tmp.length); inputOrders = tmp; } return new DualInputPlanNode(node, "CoGroup (" + node.getOperator().getName() + ")", in1, in2, DriverStrategy.CO_GROUP_RAW, this.keys1, this.keys2, inputOrders); }
@Override public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) { boolean[] inputOrders = in1.getLocalProperties().getOrdering().getFieldSortDirections(); if (inputOrders == null || inputOrders.length < this.keys1.size()) { throw new CompilerException("BUG: The input strategy does not sufficiently describe the sort orders for a merge operator."); } else if (inputOrders.length > this.keys1.size()) { boolean[] tmp = new boolean[this.keys1.size()]; System.arraycopy(inputOrders, 0, tmp, 0, tmp.length); inputOrders = tmp; } String nodeName = String.format("%s (%s)", getNodeName(), node.getOperator().getName()); return new DualInputPlanNode(node, nodeName, in1, in2, getStrategy(), this.keys1, this.keys2, inputOrders); }
/** * Parametrizes the local strategy fields of a channel such that the channel produces the desired local properties. * * @param channel The channel to parametrize. */ public void parameterizeChannel(Channel channel) { LocalProperties current = channel.getLocalProperties(); if (isMetBy(current)) { // we are met, all is good channel.setLocalStrategy(LocalStrategy.NONE); } else if (this.ordering != null) { channel.setLocalStrategy(LocalStrategy.SORT, this.ordering.getInvolvedIndexes(), this.ordering.getFieldSortDirections()); } else if (this.groupedFields != null) { boolean[] dirs = new boolean[this.groupedFields.size()]; Arrays.fill(dirs, true); channel.setLocalStrategy(LocalStrategy.SORT, Utils.createOrderedFromSet(this.groupedFields), dirs); } else { channel.setLocalStrategy(LocalStrategy.NONE); } }
if (lpp.getProperties1().isMetBy(in1.getLocalProperties()) && lpp.getProperties2().isMetBy(in2.getLocalProperties()) ) in1.getLocalProperties(), in2.getLocalProperties()))
protected void addLocalCandidates(Channel template, List<Set<? extends NamedChannel>> broadcastPlanChannels, RequestedGlobalProperties rgps, List<PlanNode> target, CostEstimator estimator) { for (RequestedLocalProperties ilp : this.inConn.getInterestingProperties().getLocalProperties()) { final Channel in = template.clone(); ilp.parameterizeChannel(in); // instantiate a candidate, if the instantiated local properties meet one possible local property set outer: for (OperatorDescriptorSingle dps: getPossibleProperties()) { for (RequestedLocalProperties ilps : dps.getPossibleLocalProperties()) { if (ilps.isMetBy(in.getLocalProperties())) { in.setRequiredLocalProps(ilps); instantiateCandidate(dps, in, broadcastPlanChannels, target, estimator, rgps, ilp); break outer; } } } } }
@Override public boolean preVisit(PlanNode visitable) { if (visitable instanceof WorksetIterationPlanNode) { PlanNode deltaNode = ((WorksetIterationPlanNode) visitable).getSolutionSetDeltaPlanNode(); //get the CoGroup DualInputPlanNode dpn = (DualInputPlanNode) deltaNode.getInputs().iterator().next().getSource(); Channel in1 = dpn.getInput1(); Channel in2 = dpn.getInput2(); Assert.assertTrue(in1.getLocalProperties().getOrdering() == null); Assert.assertTrue(in2.getLocalProperties().getOrdering() != null); Assert.assertTrue(in2.getLocalProperties().getOrdering().getInvolvedIndexes().contains(0)); Assert.assertTrue(in1.getShipStrategy() == ShipStrategyType.FORWARD); Assert.assertTrue(in2.getShipStrategy() == ShipStrategyType.PARTITION_HASH); return false; } return true; }
LocalProperties lp1 = in1.getLocalProperties().clone() .filterBySemanticProperties(semPropsLocalPropFiltering, 0); LocalProperties lp2 = in2.getLocalProperties().clone() .filterBySemanticProperties(semPropsLocalPropFiltering, 1); LocalProperties locals = operator.computeLocalProperties(lp1, lp2);
this.partialSolution.setCandidateProperties(in.getGlobalProperties(), in.getLocalProperties(), in); final BulkPartialSolutionPlanNode pspn = this.partialSolution.getCurrentPartialSolutionPlanNode(); rebuildPropertiesPlanNode.initProperties(toNoOp.getGlobalProperties(), toNoOp.getLocalProperties()); estimator.costOperator(rebuildPropertiesPlanNode);
LocalProperties lProps = in.getLocalProperties().clone(); gProps = dps.computeGlobalProperties(gProps); lProps = dps.computeLocalProperties(lProps);
this.worksetNode.setCandidateProperties(worksetIn.getGlobalProperties(), worksetIn.getLocalProperties(), worksetIn); this.solutionSetNode.setCandidateProperties(this.partitionedProperties, new LocalProperties(), solutionSetIn); toNoOp, DriverStrategy.UNARY_NO_OP); rebuildWorksetPropertiesPlanNode.initProperties(toNoOp.getGlobalProperties(), toNoOp.getLocalProperties()); estimator.costOperator(rebuildWorksetPropertiesPlanNode);
@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { Channel toReducer = in; if (in.getShipStrategy() == ShipStrategyType.FORWARD || (node.getBroadcastConnections() != null && !node.getBroadcastConnections().isEmpty())) { if (in.getSource().getOptimizerNode() instanceof PartitionNode) { LOG.warn("Cannot automatically inject combiner for ReduceFunction. Please add an explicit combiner with combineGroup() in front of the partition operator."); } } else if (combinerStrategy != DriverStrategy.NONE) { // non forward case. all local properties are killed anyways, so we can safely plug in a combiner Channel toCombiner = new Channel(in.getSource()); toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED); // create an input node for combine with same parallelism as input node ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode(); combinerNode.setParallelism(in.getSource().getParallelism()); SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getOperator().getName()+")", toCombiner, this.combinerStrategy, this.keyList); combiner.setCosts(new Costs(0, 0)); combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties()); toReducer = new Channel(combiner); toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(), in.getShipStrategySortOrder(), in.getDataExchangeMode()); toReducer.setLocalStrategy(LocalStrategy.SORT, in.getLocalStrategyKeys(), in.getLocalStrategySortOrder()); } return new SingleInputPlanNode(node, "Reduce (" + node.getOperator().getName() + ")", toReducer, DriverStrategy.SORTED_REDUCE, this.keyList); }
.getName()+")", toCombiner, DriverStrategy.SORTED_GROUP_COMBINE); combiner.setCosts(new Costs(0, 0)); combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties());
@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { if (in.getShipStrategy() == ShipStrategyType.FORWARD) { // locally connected, directly instantiate return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")", in, DriverStrategy.ALL_REDUCE); } else { // non forward case.plug in a combiner Channel toCombiner = new Channel(in.getSource()); toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED); // create an input node for combine with same parallelism as input node ReduceNode combinerNode = ((ReduceNode) node).getCombinerUtilityNode(); combinerNode.setParallelism(in.getSource().getParallelism()); SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_REDUCE); combiner.setCosts(new Costs(0, 0)); combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties()); Channel toReducer = new Channel(combiner); toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(), in.getShipStrategySortOrder(), in.getDataExchangeMode()); toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(), in.getLocalStrategySortOrder()); return new SingleInputPlanNode(node, "Reduce ("+node.getOperator().getName()+")", toReducer, DriverStrategy.ALL_REDUCE); } }
@Override public SingleInputPlanNode instantiate(Channel in, SingleInputNode node) { if (in.getShipStrategy() == ShipStrategyType.FORWARD) { // locally connected, directly instantiate return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")", in, DriverStrategy.ALL_GROUP_REDUCE); } else { // non forward case.plug in a combiner Channel toCombiner = new Channel(in.getSource()); toCombiner.setShipStrategy(ShipStrategyType.FORWARD, DataExchangeMode.PIPELINED); // create an input node for combine with same parallelism as input node GroupReduceNode combinerNode = ((GroupReduceNode) node).getCombinerUtilityNode(); combinerNode.setParallelism(in.getSource().getParallelism()); SingleInputPlanNode combiner = new SingleInputPlanNode(combinerNode, "Combine ("+node.getOperator().getName()+")", toCombiner, DriverStrategy.ALL_GROUP_REDUCE_COMBINE); combiner.setCosts(new Costs(0, 0)); combiner.initProperties(toCombiner.getGlobalProperties(), toCombiner.getLocalProperties()); Channel toReducer = new Channel(combiner); toReducer.setShipStrategy(in.getShipStrategy(), in.getShipStrategyKeys(), in.getShipStrategySortOrder(), in.getDataExchangeMode()); toReducer.setLocalStrategy(in.getLocalStrategy(), in.getLocalStrategyKeys(), in.getLocalStrategySortOrder()); return new SingleInputPlanNode(node, "GroupReduce ("+node.getOperator().getName()+")", toReducer, DriverStrategy.ALL_GROUP_REDUCE); } }
/** * Constructs a new sink candidate node that uses <i>NONE</i> as its local strategy. Note that * local sorting and range partitioning are handled by the incoming channel already. * * @param template The template optimizer node that this candidate is created for. */ public SinkPlanNode(DataSinkNode template, String nodeName, Channel input) { super(template, nodeName, input, DriverStrategy.NONE); this.globalProps = input.getGlobalProperties().clone(); this.localProps = input.getLocalProperties().clone(); }
/** * Constructs a new sink candidate node that uses <i>NONE</i> as its local strategy. Note that * local sorting and range partitioning are handled by the incoming channel already. * * @param template The template optimizer node that this candidate is created for. */ public SinkPlanNode(DataSinkNode template, String nodeName, Channel input) { super(template, nodeName, input, DriverStrategy.NONE); this.globalProps = input.getGlobalProperties().clone(); this.localProps = input.getLocalProperties().clone(); }
@Override public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) { boolean[] inputOrders = in1.getLocalProperties().getOrdering() == null ? null : in1.getLocalProperties().getOrdering().getFieldSortDirections(); if (inputOrders == null || inputOrders.length < this.keys1.size()) { throw new CompilerException("BUG: The input strategy does not sufficiently describe the sort orders for a CoGroup operator."); } else if (inputOrders.length > this.keys1.size()) { boolean[] tmp = new boolean[this.keys1.size()]; System.arraycopy(inputOrders, 0, tmp, 0, tmp.length); inputOrders = tmp; } return new DualInputPlanNode(node, "CoGroup ("+node.getOperator().getName()+")", in1, in2, DriverStrategy.CO_GROUP, this.keys1, this.keys2, inputOrders); }