/** * Adds a new outgoing connection to this node. * * @param connection * The connection to add. */ public void addOutgoingConnection(DagConnection connection) { if (this.outgoingConnections == null) { this.outgoingConnections = new ArrayList<DagConnection>(); } else { if (this.outgoingConnections.size() == 64) { throw new CompilerException("Cannot currently handle nodes with more than 64 outputs."); } } this.outgoingConnections.add(connection); }
public static boolean[] getDirections(Ordering o, int numFields) { final boolean[] dirs = o.getFieldSortDirections(); if (dirs.length == numFields) { return dirs; } else if (dirs.length > numFields) { final boolean[] subSet = new boolean[numFields]; System.arraycopy(dirs, 0, subSet, 0, numFields); return subSet; } else { throw new CompilerException(); } }
@Override public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) { if (node instanceof SinkJoiner) { return new SinkJoinerPlanNode((SinkJoiner) node, in1, in2); } else { throw new CompilerException(); } }
private void computeLocalPropertiesAfterShippingOnly() { switch (this.shipStrategy) { case BROADCAST: case PARTITION_HASH: case PARTITION_CUSTOM: case PARTITION_RANGE: case PARTITION_RANDOM: case PARTITION_FORCED_REBALANCE: this.localProps = new LocalProperties(); break; case FORWARD: this.localProps = this.source.getLocalProperties(); break; case NONE: throw new CompilerException("ShipStrategy has not yet been set."); default: throw new CompilerException("Unknown ShipStrategy."); } }
@Override public void postVisit(OptimizerNode visitable) { visitable.identifyDynamicPath(this.costWeight); // check that there is no nested iteration on the dynamic path if (visitable.isOnDynamicPath() && visitable instanceof IterationNode) { throw new CompilerException("Nested iterations are currently not supported."); } } }
/** * Creates a new node for the bulk iteration. * * @param iteration The bulk iteration the node represents. */ public BulkIterationNode(BulkIterationBase<?> iteration) { super(iteration); if (iteration.getMaximumNumberOfIterations() <= 0) { throw new CompilerException("BulkIteration must have a maximum number of iterations specified."); } int numIters = iteration.getMaximumNumberOfIterations(); this.costWeight = (numIters > 0 && numIters < OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT) ? numIters : OptimizerNode.MAX_DYNAMIC_PATH_COST_WEIGHT; }
@Override public void addOutgoingConnection(DagConnection connection) { // ensure that union nodes have not more than one outgoing connection. if (this.getOutgoingConnections() != null && this.getOutgoingConnections().size() > 0) { throw new CompilerException("BinaryUnionNode may only have a single outgoing connection."); } super.addOutgoingConnection(connection); }
/** * Sets the key field information for the specified driver comparator. * * @param keys The key field indexes for the specified driver comparator. * @param sortOrder The key sort order for the specified driver comparator. * @param id The ID of the driver comparator. */ public void setDriverKeyInfo(FieldList keys, boolean[] sortOrder, int id) { if(id < 0 || id >= driverKeys.length) { throw new CompilerException("Invalid id for driver key information. DriverStrategy requires only " +super.getDriverStrategy().getNumRequiredComparators()+" comparators."); } this.driverKeys[id] = keys; this.driverSortOrders[id] = sortOrder; }
public void collect(Channel in, List<Channel> inputs) { if (in.getSource() instanceof NAryUnionPlanNode) { // sanity check if (in.getShipStrategy() != ShipStrategyType.FORWARD) { throw new CompilerException("Bug: Plan generation for Unions picked a ship strategy between binary plan operators."); } if (!(in.getLocalStrategy() == null || in.getLocalStrategy() == LocalStrategy.NONE)) { throw new CompilerException("Bug: Plan generation for Unions picked a local strategy between binary plan operators."); } inputs.addAll(((NAryUnionPlanNode) in.getSource()).getListOfInputs()); } else { // is not a collapsed union node, so we take the channel directly inputs.add(in); } } }
public LocalProperties getLocalProperties() { if (this.localProps == null) { computeLocalPropertiesAfterShippingOnly(); switch (this.localStrategy) { case NONE: break; case SORT: case COMBININGSORT: this.localProps = LocalProperties.forOrdering(Utils.createOrdering(this.localKeys, this.localSortOrder)); break; default: throw new CompilerException("Unsupported local strategy for channel."); } } return this.localProps; }
@Override public void accept(Visitor<OptimizerNode> visitor) { if (visitor.preVisit(this)) { if (getPredecessorNode() != null) { getPredecessorNode().accept(visitor); } else { throw new CompilerException(); } visitor.postVisit(this); } } }
@Override public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) { boolean[] inputOrders = in1.getLocalProperties().getOrdering().getFieldSortDirections(); if (inputOrders == null || inputOrders.length < this.keys1.size()) { throw new CompilerException("BUG: The input strategy does not sufficiently describe the sort orders for a merge operator."); } else if (inputOrders.length > this.keys1.size()) { boolean[] tmp = new boolean[this.keys1.size()]; System.arraycopy(inputOrders, 0, tmp, 0, tmp.length); inputOrders = tmp; } String nodeName = String.format("%s (%s)", getNodeName(), node.getOperator().getName()); return new DualInputPlanNode(node, nodeName, in1, in2, getStrategy(), this.keys1, this.keys2, inputOrders); }
@Override public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) { boolean[] inputOrders = in2.getLocalProperties().getOrdering() == null ? null : in2.getLocalProperties().getOrdering().getFieldSortDirections(); if (inputOrders == null || inputOrders.length < this.keys2.size()) { throw new CompilerException("BUG: The input strategy does not sufficiently describe the sort orders for a CoGroup operator."); } else if (inputOrders.length > this.keys2.size()) { boolean[] tmp = new boolean[this.keys2.size()]; System.arraycopy(inputOrders, 0, tmp, 0, tmp.length); inputOrders = tmp; } return new DualInputPlanNode(node, "CoGroup ("+node.getOperator().getName()+")", in1, in2, DriverStrategy.CO_GROUP, this.keys1, this.keys2, inputOrders); }
@Override public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) { boolean[] inputOrders = in1.getLocalProperties().getOrdering() == null ? null : in1.getLocalProperties().getOrdering().getFieldSortDirections(); if (inputOrders == null || inputOrders.length < this.keys1.size()) { throw new CompilerException("BUG: The input strategy does not sufficiently describe the sort orders for a CoGroup operator."); } else if (inputOrders.length > this.keys1.size()) { boolean[] tmp = new boolean[this.keys1.size()]; System.arraycopy(inputOrders, 0, tmp, 0, tmp.length); inputOrders = tmp; } return new DualInputPlanNode(node, "CoGroup ("+node.getOperator().getName()+")", in1, in2, DriverStrategy.CO_GROUP, this.keys1, this.keys2, inputOrders); }
@Override public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) { boolean[] inputOrders = in1.getLocalProperties().getOrdering() == null ? null : in1.getLocalProperties().getOrdering().getFieldSortDirections(); if (inputOrders == null || inputOrders.length < this.keys1.size()) { throw new CompilerException("BUG: The input strategy does not sufficiently describe the sort orders for a CoGroup operator."); } else if (inputOrders.length > this.keys1.size()) { boolean[] tmp = new boolean[this.keys1.size()]; System.arraycopy(inputOrders, 0, tmp, 0, tmp.length); inputOrders = tmp; } return new DualInputPlanNode(node, "CoGroup (" + node.getOperator().getName() + ")", in1, in2, DriverStrategy.CO_GROUP_RAW, this.keys1, this.keys2, inputOrders); }
@Override public boolean areCoFulfilled(RequestedLocalProperties requested1, RequestedLocalProperties requested2, LocalProperties produced1, LocalProperties produced2) { int numRelevantFields = this.keys1.size(); Ordering prod1 = produced1.getOrdering(); Ordering prod2 = produced2.getOrdering(); if (prod1 == null || prod2 == null || prod1.getNumberOfFields() < numRelevantFields || prod2.getNumberOfFields() < numRelevantFields) { throw new CompilerException("The given properties do not meet this operators requirements."); } for (int i = 0; i < numRelevantFields; i++) { if (prod1.getOrder(i) != prod2.getOrder(i)) { return false; } } return true; }
@Override public void accept(Visitor<OptimizerNode> visitor) { if (visitor.preVisit(this)) { if (getPredecessorNode() != null) { getPredecessorNode().accept(visitor); } else { throw new CompilerException(); } for (DagConnection connection : getBroadcastConnections()) { connection.getSource().accept(visitor); } visitor.postVisit(this); } } }
@Override public void accept(Visitor<OptimizerNode> visitor) { if (visitor.preVisit(this)) { if (this.input1 == null || this.input2 == null) { throw new CompilerException(); } getFirstPredecessorNode().accept(visitor); getSecondPredecessorNode().accept(visitor); for (DagConnection connection : getBroadcastConnections()) { connection.getSource().accept(visitor); } visitor.postVisit(this); } } }
@Override public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) { DriverStrategy strategy; if (!in2.isOnDynamicPath() && in1.isOnDynamicPath()) { // sanity check that the first input is cached and remove that cache if (!in2.getTempMode().isCached()) { throw new CompilerException("No cache at point where static and dynamic parts meet."); } in2.setTempMode(in2.getTempMode().makeNonCached()); strategy = DriverStrategy.HYBRIDHASH_BUILD_SECOND_CACHED; } else { strategy = DriverStrategy.HYBRIDHASH_BUILD_SECOND; } return new DualInputPlanNode(node, "Join ("+node.getOperator().getName()+")", in1, in2, strategy, this.keys1, this.keys2); }
@Override public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) { DriverStrategy strategy; if(!in1.isOnDynamicPath() && in2.isOnDynamicPath()) { // sanity check that the first input is cached and remove that cache if (!in1.getTempMode().isCached()) { throw new CompilerException("No cache at point where static and dynamic parts meet."); } in1.setTempMode(in1.getTempMode().makeNonCached()); strategy = DriverStrategy.HYBRIDHASH_BUILD_FIRST_CACHED; } else { strategy = DriverStrategy.HYBRIDHASH_BUILD_FIRST; } return new DualInputPlanNode(node, "Join ("+node.getOperator().getName()+")", in1, in2, strategy, this.keys1, this.keys2); }