/** * Returns a {@link JobVertexID} to vertex name mapping for the given graph. */ private Map<JobVertexID, String> rememberIds(JobGraph jobGraph) { final Map<JobVertexID, String> ids = new HashMap<>(); for (JobVertex vertex : jobGraph.getVertices()) { ids.put(vertex.getID(), vertex.getName()); } return ids; }
/** * Verifies that no {@link JobVertexID} of the {@link JobGraph} is contained in the given map. */ private void verifyIdsNotEqual(JobGraph jobGraph, Map<JobVertexID, String> ids) { // Verify same number of vertices assertEquals(jobGraph.getNumberOfVertices(), ids.size()); // Verify that all IDs->name mappings are identical for (JobVertex vertex : jobGraph.getVertices()) { assertFalse(ids.containsKey(vertex.getID())); } }
/** * Verifies that each {@link JobVertexID} of the {@link JobGraph} is contained in the given map * and mapped to the same vertex name. */ private void verifyIdsEqual(JobGraph jobGraph, Map<JobVertexID, String> ids) { // Verify same number of vertices assertEquals(jobGraph.getNumberOfVertices(), ids.size()); // Verify that all IDs->name mappings are identical for (JobVertex vertex : jobGraph.getVertices()) { String expectedName = ids.get(vertex.getID()); assertNotNull(expectedName); assertEquals(expectedName, vertex.getName()); } }
@Test public void testUserProvidedHashing() { StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(); List<String> userHashes = Arrays.asList("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"); env.addSource(new NoOpSourceFunction(), "src").setUidHash(userHashes.get(0)) .map(new NoOpMapFunction()) .filter(new NoOpFilterFunction()) .keyBy(new NoOpKeySelector()) .reduce(new NoOpReduceFunction()).name("reduce").setUidHash(userHashes.get(1)); StreamGraph streamGraph = env.getStreamGraph(); int idx = 1; for (JobVertex jobVertex : streamGraph.getJobGraph().getVertices()) { List<JobVertexID> idAlternatives = jobVertex.getIdAlternatives(); Assert.assertEquals(idAlternatives.get(idAlternatives.size() - 1).toString(), userHashes.get(idx)); --idx; } }
@Test public void testCheckpointModeTranslation() { try { // with deactivated fault tolerance, the checkpoint mode should be at-least-once StreamExecutionEnvironment deactivated = getSimpleJob(); for (JobVertex vertex : deactivated.getStreamGraph().getJobGraph().getVertices()) { assertEquals(CheckpointingMode.AT_LEAST_ONCE, new StreamConfig(vertex.getConfiguration()).getCheckpointMode()); } // with activated fault tolerance, the checkpoint mode should be by default exactly once StreamExecutionEnvironment activated = getSimpleJob(); activated.enableCheckpointing(1000L); for (JobVertex vertex : activated.getStreamGraph().getJobGraph().getVertices()) { assertEquals(CheckpointingMode.EXACTLY_ONCE, new StreamConfig(vertex.getConfiguration()).getCheckpointMode()); } // explicitly setting the mode StreamExecutionEnvironment explicit = getSimpleJob(); explicit.enableCheckpointing(1000L, CheckpointingMode.AT_LEAST_ONCE); for (JobVertex vertex : explicit.getStreamGraph().getJobGraph().getVertices()) { assertEquals(CheckpointingMode.AT_LEAST_ONCE, new StreamConfig(vertex.getConfiguration()).getCheckpointMode()); } } catch (Exception e) { e.printStackTrace(); fail(e.getMessage()); } }
/** * Tests that there are no collisions with two identical intermediate nodes connected to the * same predecessor. * * <pre> * /-> [ (map) ] -> [ (sink) ] * [ (src) ] -+ * \-> [ (map) ] -> [ (sink) ] * </pre> */ @Test public void testNodeHashIdenticalNodes() throws Exception { StreamExecutionEnvironment env = StreamExecutionEnvironment.createLocalEnvironment(); env.setParallelism(4); env.disableOperatorChaining(); DataStream<String> src = env.addSource(new NoOpSourceFunction()); src.map(new NoOpMapFunction()).addSink(new NoOpSinkFunction()); src.map(new NoOpMapFunction()).addSink(new NoOpSinkFunction()); JobGraph jobGraph = env.getStreamGraph().getJobGraph(); Set<JobVertexID> vertexIds = new HashSet<>(); for (JobVertex vertex : jobGraph.getVertices()) { assertTrue(vertexIds.add(vertex.getID())); } }
for (JobVertex jobVertex : jobGraph.getVertices()) { if (jobVertex.getName().contains("test_source")) { assertTrue(jobVertex.getMinResources().equals(resource1));
JobVertex itSink1 = null; for (JobVertex vertex : jg.getVertices()) { if (vertex.getName().contains("IterationSource")) { itSource1 = vertex;
JobVertex itSink1 = null; for (JobVertex vertex : jg.getVertices()) { if (vertex.getName().contains("IterationSource")) { itSource1 = vertex;
for (JobVertex vertex : jobGraph.getVertices()) { assertTrue(ids.add(vertex.getID())); assertNotEquals(jobGraph.getJobID(), newJobGraph.getJobID()); for (JobVertex vertex : newJobGraph.getVertices()) {
@Test public void testTempInIterationTest() throws Exception { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Tuple2<Long, Long>> input = env.readCsvFile("file:///does/not/exist").types(Long.class, Long.class); DeltaIteration<Tuple2<Long, Long>, Tuple2<Long, Long>> iteration = input.iterateDelta(input, 1, 0); DataSet<Tuple2<Long, Long>> update = iteration.getWorkset() .join(iteration.getSolutionSet()).where(0).equalTo(0) .with(new DummyFlatJoinFunction<Tuple2<Long, Long>>()); iteration.closeWith(update, update) .output(new DiscardingOutputFormat<Tuple2<Long, Long>>()); Plan plan = env.createProgramPlan(); OptimizedPlan oPlan = (new Optimizer(new Configuration())).compile(plan); JobGraphGenerator jgg = new JobGraphGenerator(); JobGraph jg = jgg.compileJobGraph(oPlan); boolean solutionSetUpdateChecked = false; for(JobVertex v : jg.getVertices()) { if(v.getName().equals("SolutionSet Delta")) { // check if input of solution set delta is temped TaskConfig tc = new TaskConfig(v.getConfiguration()); assertTrue(tc.isInputAsynchronouslyMaterialized(0)); solutionSetUpdateChecked = true; } } assertTrue(solutionSetUpdateChecked); }
public IntermediateDataSet getResult(IntermediateDataSetID resultID) { // Lazy build the result map if the result is not found, as edges may be late added if (!results.containsKey(resultID)) { for (JobVertex vertex : getVertices()) { for (IntermediateDataSet result : vertex.getProducedDataSets()) { results.put(result.getId(), result); } } } return results.get(resultID); }
public Collection<IntermediateDataSetID> getResultIDs() { // Lazy build the result map if the result is not found, as edges may be late added for (JobVertex vertex : getVertices()) { for (IntermediateDataSet result : vertex.getProducedDataSets()) { results.put(result.getId(), result); } } return results.keySet(); }
@Override public CompletableFuture<Acknowledge> rescaleJob( int newParallelism, RescalingBehaviour rescalingBehaviour, Time timeout) { final ArrayList<JobVertexID> allOperators = new ArrayList<>(jobGraph.getNumberOfVertices()); for (JobVertex jobVertex : jobGraph.getVertices()) { allOperators.add(jobVertex.getID()); } return rescaleOperators(allOperators, newParallelism, rescalingBehaviour, timeout); }
@Override public CompletableFuture<Acknowledge> rescaleJob( int newParallelism, RescalingBehaviour rescalingBehaviour, Time timeout) { final ArrayList<JobVertexID> allOperators = new ArrayList<>(jobGraph.getNumberOfVertices()); for (JobVertex jobVertex : jobGraph.getVertices()) { allOperators.add(jobVertex.getID()); } return rescaleOperators(allOperators, newParallelism, rescalingBehaviour, timeout); }
@Override public CompletableFuture<Acknowledge> rescaleJob( int newParallelism, RescalingBehaviour rescalingBehaviour, Time timeout) { final ArrayList<JobVertexID> allOperators = new ArrayList<>(jobGraph.getNumberOfVertices()); for (JobVertex jobVertex : jobGraph.getVertices()) { allOperators.add(jobVertex.getID()); } return rescaleOperators(allOperators, newParallelism, rescalingBehaviour, timeout); }
@Override public void updateJobGraph(JobGraph jobGraph) { for (JobVertex vertex : jobGraph.getVertices()) { if (jobVertexID.equals(vertex.getID())) { updateVertex(vertex); } } } }
for (JobVertex vertex : jg.getVertices()) {
for (JobVertex jobVertex : jobGraph.getVertices()) { slotsCount += jobVertex.getParallelism();
for (JobVertex vertex : jg.getVertices()) {