private int getNumberOfSendersPerReceiver(DistributionPattern pattern, int numSenders, int numReceivers) { if (pattern == DistributionPattern.ALL_TO_ALL) { return numSenders; } else if (pattern == DistributionPattern.POINTWISE) { if (numSenders != numReceivers) { if (numReceivers == 1) { return numSenders; } else if (numSenders == 1) { return 1; } else { throw new CompilerException("Error: A changing parallelism is currently " + "not supported between tasks within an iteration."); } } else { return 1; } } else { throw new CompilerException("Unknown distribution pattern for channels: " + pattern); } }
@Test public void testReduce() { // construct the plan ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(DEFAULT_PARALLELISM); DataSet<Long> set1 = env.generateSequence(0,1); set1.reduceGroup(new IdentityGroupReducer<Long>()).name("Reduce1") .output(new DiscardingOutputFormat<Long>()).name("Sink"); Plan plan = env.createProgramPlan(); try { OptimizedPlan oPlan = compileNoStats(plan); JobGraphGenerator jobGen = new JobGraphGenerator(); jobGen.compileJobGraph(oPlan); } catch(CompilerException ce) { ce.printStackTrace(); fail("The pact compiler is unable to compile this plan correctly"); } } }
assertTrue(e.getMessage().toLowerCase().indexOf("nested iterations") != -1);
oPlan = compileNoStats(plan); } catch(CompilerException ce) { ce.printStackTrace(); fail("The pact compiler is unable to compile this plan correctly."); return; // silence the compiler
assertTrue(e.getMessage().toLowerCase().indexOf("nested iterations") != -1);
/** * Adds a new outgoing connection to this node. * * @param connection * The connection to add. */ public void addOutgoingConnection(DagConnection connection) { if (this.outgoingConnections == null) { this.outgoingConnections = new ArrayList<DagConnection>(); } else { if (this.outgoingConnections.size() == 64) { throw new CompilerException("Cannot currently handle nodes with more than 64 outputs."); } } this.outgoingConnections.add(connection); }
@Test public void testCrossWithSmall() { // construct the plan ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(DEFAULT_PARALLELISM); DataSet<Long> set1 = env.generateSequence(0,1); DataSet<Long> set2 = env.generateSequence(0,1); set1.crossWithTiny(set2).name("Cross") .output(new DiscardingOutputFormat<Tuple2<Long, Long>>()); try { Plan plan = env.createProgramPlan(); OptimizedPlan oPlan = compileWithStats(plan); OptimizerPlanNodeResolver resolver = new OptimizerPlanNodeResolver(oPlan); DualInputPlanNode crossPlanNode = resolver.getNode("Cross"); Channel in1 = crossPlanNode.getInput1(); Channel in2 = crossPlanNode.getInput2(); assertEquals(ShipStrategyType.FORWARD, in1.getShipStrategy()); assertEquals(ShipStrategyType.BROADCAST, in2.getShipStrategy()); } catch(CompilerException ce) { ce.printStackTrace(); fail("The Flink optimizer is unable to compile this plan correctly."); } }
oPlan = compileNoStats(plan); } catch(CompilerException e) { Assert.fail(e.getMessage());
public static boolean[] getDirections(Ordering o, int numFields) { final boolean[] dirs = o.getFieldSortDirections(); if (dirs.length == numFields) { return dirs; } else if (dirs.length > numFields) { final boolean[] subSet = new boolean[numFields]; System.arraycopy(dirs, 0, subSet, 0, numFields); return subSet; } else { throw new CompilerException(); } }
@Test public void testCrossWithLarge() { // construct the plan ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.setParallelism(DEFAULT_PARALLELISM); DataSet<Long> set1 = env.generateSequence(0,1); DataSet<Long> set2 = env.generateSequence(0,1); set1.crossWithHuge(set2).name("Cross") .output(new DiscardingOutputFormat<Tuple2<Long, Long>>()); try { Plan plan = env.createProgramPlan(); OptimizedPlan oPlan = compileNoStats(plan); OptimizerPlanNodeResolver resolver = new OptimizerPlanNodeResolver(oPlan); DualInputPlanNode crossPlanNode = resolver.getNode("Cross"); Channel in1 = crossPlanNode.getInput1(); Channel in2 = crossPlanNode.getInput2(); assertEquals(ShipStrategyType.BROADCAST, in1.getShipStrategy()); assertEquals(ShipStrategyType.FORWARD, in2.getShipStrategy()); } catch(CompilerException ce) { ce.printStackTrace(); fail("The pact compiler is unable to compile this plan correctly."); } } }
@Override public DualInputPlanNode instantiate(Channel in1, Channel in2, TwoInputNode node) { if (node instanceof SinkJoiner) { return new SinkJoinerPlanNode((SinkJoiner) node, in1, in2); } else { throw new CompilerException(); } }
oPlan = compileNoStats(plan); } catch(CompilerException ce) { ce.printStackTrace(); fail("The pact compiler is unable to compile this plan correctly."); return; // silence the compiler
public void adjustGlobalPropertiesForFullParallelismChange() { if (this.shipStrategy == null || this.shipStrategy == ShipStrategyType.NONE) { throw new IllegalStateException("Cannot adjust channel for parallelism " + "change before the ship strategy is set."); } // make sure the properties are acquired if (this.globalProps == null) { getGlobalProperties(); } // some strategies globally reestablish properties switch (this.shipStrategy) { case FORWARD: throw new CompilerException("Cannot use FORWARD strategy between operations " + "with different number of parallel instances."); case NONE: // excluded by sanity check. left here for verification check completion case BROADCAST: case PARTITION_HASH: case PARTITION_RANGE: case PARTITION_RANDOM: case PARTITION_FORCED_REBALANCE: case PARTITION_CUSTOM: return; } throw new CompilerException("Unrecognized Ship Strategy Type: " + this.shipStrategy); }
oPlan = compileNoStats(plan); } catch(CompilerException ce) { ce.printStackTrace(); fail("The pact compiler is unable to compile this plan correctly."); return; // silence the compiler
private OptimizerPostPass getPostPassFromPlan(Plan program) { final String className = program.getPostPassClassName(); if (className == null) { throw new CompilerException("Optimizer Post Pass class description is null"); } try { Class<? extends OptimizerPostPass> clazz = Class.forName(className).asSubclass(OptimizerPostPass.class); try { return InstantiationUtil.instantiate(clazz, OptimizerPostPass.class); } catch (RuntimeException rtex) { // unwrap the source exception if (rtex.getCause() != null) { throw new CompilerException("Cannot instantiate optimizer post pass: " + rtex.getMessage(), rtex.getCause()); } else { throw rtex; } } } catch (ClassNotFoundException cnfex) { throw new CompilerException("Cannot load Optimizer post-pass class '" + className + "'.", cnfex); } catch (ClassCastException ccex) { throw new CompilerException("Class '" + className + "' is not an optimizer post-pass.", ccex); } } }
oPlan = compileNoStats(plan); } catch(CompilerException ce) { ce.printStackTrace(); fail("The pact compiler is unable to compile this plan correctly."); return; // silence the compiler
private void computeLocalPropertiesAfterShippingOnly() { switch (this.shipStrategy) { case BROADCAST: case PARTITION_HASH: case PARTITION_CUSTOM: case PARTITION_RANGE: case PARTITION_RANDOM: case PARTITION_FORCED_REBALANCE: this.localProps = new LocalProperties(); break; case FORWARD: this.localProps = this.source.getLocalProperties(); break; case NONE: throw new CompilerException("ShipStrategy has not yet been set."); default: throw new CompilerException("Unknown ShipStrategy."); } }
oPlan = compileNoStats(plan); } catch(CompilerException ce) { ce.printStackTrace(); fail("The pact compiler is unable to compile this plan correctly."); return; // silence the compiler
throw new CompilerException("Error in compiler: " + "Parent to get branch info for is not contained in the outgoing connections."); throw new CompilerException( "Error in compiler: Cannot get branch info for successor in a node with no successors.");
private void mergeBranchPlanMaps() { for (OptimizerNode.UnclosedBranchDescriptor desc: template.getOpenBranches()) { OptimizerNode brancher = desc.getBranchingNode(); if (branchPlan == null) { branchPlan = new HashMap<OptimizerNode, PlanNode>(6); } if (!branchPlan.containsKey(brancher)) { PlanNode selectedCandidate = null; if (rootOfStepFunction.branchPlan != null) { selectedCandidate = rootOfStepFunction.branchPlan.get(brancher); } if (selectedCandidate == null) { throw new CompilerException( "Candidates for a node with open branches are missing information about the selected candidate "); } this.branchPlan.put(brancher, selectedCandidate); } } } }