return new Tuple2<Long, Long>(left.f0, right.f1); }).withForwardedFieldsFirst("0").withForwardedFieldsSecond("1") .union(paths) .groupBy(0, 1)
@Test public void testBinaryForwardedInLine1() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(3L, 4L)); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input2 = env.fromElements(new Tuple2<Long, Long>(3L, 2L)); input1.join(input2).where(0).equalTo(0).with(new NoAnnotationJoin<Long>()) .withForwardedFieldsFirst("0->1; 1->2").withForwardedFieldsSecond("1->0") .output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>()); Plan plan = env.createProgramPlan(); GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next(); InnerJoinOperatorBase<?, ?, ?, ?> join = (InnerJoinOperatorBase<?, ?, ?, ?>) sink.getInput(); DualInputSemanticProperties semantics = join.getSemanticProperties(); assertNotNull(semantics.getForwardingTargetFields(1, 0)); assertEquals(1, semantics.getForwardingTargetFields(0, 0).size()); assertEquals(1, semantics.getForwardingTargetFields(0, 1).size()); assertEquals(1, semantics.getForwardingTargetFields(1, 1).size()); assertTrue(semantics.getForwardingTargetFields(0, 0).contains(1)); assertTrue(semantics.getForwardingTargetFields(0, 1).contains(2)); assertTrue(semantics.getForwardingTargetFields(1, 1).contains(0)); assertEquals(0, semantics.getForwardingTargetFields(1, 0).size()); }
.join(this.vertices).where(0).equalTo(0) .with(new ProjectVertexWithNeighborValueJoin<>(1)) .withForwardedFieldsFirst("f1->f0").name("Vertex with in-neighbor value"); return verticesWithSourceNeighborValues.groupBy(0).reduce(new ApplyNeighborReduceFunction<>( reduceNeighborsFunction)).name("Neighbors function"); .join(this.vertices).where(1).equalTo(0) .with(new ProjectVertexWithNeighborValueJoin<>(0)) .withForwardedFieldsFirst("f0").name("Vertex with out-neighbor value"); return verticesWithTargetNeighborValues.groupBy(0).reduce(new ApplyNeighborReduceFunction<>( reduceNeighborsFunction)).name("Neighbors function");
/** * Computes new vertices based on the new subgraphs. For each vertex, each * graph is collected in a flatMap function and then joined with the new * subgraph dataset. * * @param newGraphHeads graph dataset of the resulting graph collection * @return vertex set of the resulting graph collection */ @Override protected DataSet<Vertex> computeNewVertices( DataSet<GraphHead> newGraphHeads) { DataSet<Tuple2<Vertex, GradoopId>> verticesWithGraphs = firstCollection.getVertices().flatMap(new PairVertexWithGraphs<>()); return verticesWithGraphs .join(newGraphHeads) .where(1) .equalTo(new Id<>()) .with(new LeftJoin0OfTuple2<>()) .withForwardedFieldsFirst("f0->*") .distinct(new Id<>()); }
/** * Computes the intersection between the edge set and the given edge set. For all matching pairs, * only one edge will be in the resulting data set. * * @param edges edges to compute intersection with * @return edge set containing one edge for all matching pairs of the same edge */ private DataSet<Edge<K, EV>> getDistinctEdgeIntersection(DataSet<Edge<K, EV>> edges) { return this.getEdges() .join(edges) .where(0, 1, 2) .equalTo(0, 1, 2) .with(new JoinFunction<Edge<K, EV>, Edge<K, EV>, Edge<K, EV>>() { @Override public Edge<K, EV> join(Edge<K, EV> first, Edge<K, EV> second) throws Exception { return first; } }).withForwardedFieldsFirst("*").name("Intersect edges") .distinct() .name("Edges"); }
/** * Computes new vertices based on the new subgraphs. For each vertex, each * graph is collected in a flatMap function and then joined with the new * subgraph dataset. * * @param newGraphHeads graph dataset of the resulting graph collection * @return vertex set of the resulting graph collection */ @Override protected DataSet<Vertex> computeNewVertices( DataSet<GraphHead> newGraphHeads) { DataSet<Tuple2<Vertex, GradoopId>> verticesWithGraphs = firstCollection.getVertices().flatMap(new PairVertexWithGraphs<>()); return verticesWithGraphs .join(newGraphHeads) .where(1) .equalTo(new Id<>()) .with(new LeftJoin0OfTuple2<>()) .withForwardedFieldsFirst("f0->*") .distinct(new Id<>()); }
/** * Computes the intersection between the edge set and the given edge set. For all matching pairs, * only one edge will be in the resulting data set. * * @param edges edges to compute intersection with * @return edge set containing one edge for all matching pairs of the same edge */ private DataSet<Edge<K, EV>> getDistinctEdgeIntersection(DataSet<Edge<K, EV>> edges) { return this.getEdges() .join(edges) .where(0, 1, 2) .equalTo(0, 1, 2) .with(new JoinFunction<Edge<K, EV>, Edge<K, EV>, Edge<K, EV>>() { @Override public Edge<K, EV> join(Edge<K, EV> first, Edge<K, EV> second) throws Exception { return first; } }).withForwardedFieldsFirst("*").name("Intersect edges") .distinct() .name("Edges"); }
/** * Computes the intersection between the edge set and the given edge set. For all matching pairs, * only one edge will be in the resulting data set. * * @param edges edges to compute intersection with * @return edge set containing one edge for all matching pairs of the same edge */ private DataSet<Edge<K, EV>> getDistinctEdgeIntersection(DataSet<Edge<K, EV>> edges) { return this.getEdges() .join(edges) .where(0, 1, 2) .equalTo(0, 1, 2) .with(new JoinFunction<Edge<K, EV>, Edge<K, EV>, Edge<K, EV>>() { @Override public Edge<K, EV> join(Edge<K, EV> first, Edge<K, EV> second) throws Exception { return first; } }).withForwardedFieldsFirst("*").name("Intersect edges") .distinct() .name("Edges"); }
out.collect(new Tuple3<>(first.f0, first.f1, second.f1)); }).withForwardedFieldsFirst("f0;f1").withForwardedFieldsSecond("f1"); new Tuple3<>(vertex.getValue(), degrees.f1, degrees.f2))); }).withForwardedFieldsFirst("f0");
.withForwardedFieldsFirst("0") .withForwardedFieldsSecond("0") .name("Update Component Id"); .withForwardedFieldsFirst("0") .withForwardedFieldsSecond("0") .name("Update Component Id");
.join(this.vertices).where(0).equalTo(0) .with(new ProjectVertexIdJoin<>(1)) .withForwardedFieldsFirst("f1->f0").name("Edge with source vertex ID"); return edgesWithSources.groupBy(0).reduceGroup( new ApplyNeighborGroupReduceFunction<>(neighborsFunction)) .join(this.vertices).where(1).equalTo(0) .with(new ProjectVertexIdJoin<>(0)) .withForwardedFieldsFirst("f0").name("Edge with target vertex ID"); return edgesWithTargets.groupBy(0).reduceGroup( new ApplyNeighborGroupReduceFunction<>(neighborsFunction))
.join(this.vertices).where(0).equalTo(0) .with(new ProjectVertexIdJoin<>(1)) .withForwardedFieldsFirst("f1->f0").name("Edge with source vertex ID"); return edgesWithSources.groupBy(0).reduceGroup( new ApplyNeighborGroupReduceFunction<>(neighborsFunction)).name("Neighbors function"); .join(this.vertices).where(1).equalTo(0) .with(new ProjectVertexIdJoin<>(0)) .withForwardedFieldsFirst("f0").name("Edge with target vertex ID"); return edgesWithTargets.groupBy(0).reduceGroup( new ApplyNeighborGroupReduceFunction<>(neighborsFunction)).name("Neighbors function");
.where(new DummyTestKeySelector()).equalTo(new DummyTestKeySelector()) .with(new DummyTestJoinFunction2()) .withForwardedFieldsFirst("2;4->0") .withForwardedFieldsSecond("0->4;1;1->3");
.withForwardedFieldsFirst("*").name(JOIN_WITH_INVARIANT_NAME);
@Test public void testBinaryForwardedInLine2() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(3L, 4L)); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input2 = env.fromElements(new Tuple2<Long, Long>(3L, 2L)); input1.join(input2).where(0).equalTo(0).with(new ReadSetJoin<Long>()) .withForwardedFieldsFirst("0->1; 1->2").withForwardedFieldsSecond("1->0") .output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>()); Plan plan = env.createProgramPlan(); GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next(); InnerJoinOperatorBase<?, ?, ?, ?> join = (InnerJoinOperatorBase<?, ?, ?, ?>) sink.getInput(); DualInputSemanticProperties semantics = join.getSemanticProperties(); assertNotNull(semantics.getForwardingTargetFields(1, 0)); assertEquals(1, semantics.getForwardingTargetFields(0, 0).size()); assertEquals(1, semantics.getForwardingTargetFields(0, 1).size()); assertEquals(1, semantics.getForwardingTargetFields(1, 1).size()); assertTrue(semantics.getForwardingTargetFields(0, 0).contains(1)); assertTrue(semantics.getForwardingTargetFields(0, 1).contains(2)); assertTrue(semantics.getForwardingTargetFields(1, 1).contains(0)); assertNotNull(semantics.getReadFields(0)); assertNotNull(semantics.getReadFields(1)); assertEquals(1, semantics.getReadFields(0).size()); assertEquals(1, semantics.getReadFields(1).size()); assertTrue(semantics.getReadFields(0).contains(1)); assertTrue(semantics.getReadFields(1).contains(0)); assertEquals(0, semantics.getForwardingTargetFields(1, 0).size()); }
@Test public void testBinaryForwardedAnnotationInLineMixed2() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(3L, 4L)); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input2 = env.fromElements(new Tuple2<Long, Long>(3L, 2L)); input1.join(input2).where(0).equalTo(0).with(new ForwardedSecondAnnotationJoin<Long>()) .withForwardedFieldsFirst("0->1") .output(new DiscardingOutputFormat<Tuple3<Long, Long, Long>>()); Plan plan = env.createProgramPlan(); GenericDataSinkBase<?> sink = plan.getDataSinks().iterator().next(); InnerJoinOperatorBase<?, ?, ?, ?> join = (InnerJoinOperatorBase<?, ?, ?, ?>) sink.getInput(); DualInputSemanticProperties semantics = join.getSemanticProperties(); assertNotNull(semantics.getForwardingTargetFields(0, 1)); assertNotNull(semantics.getForwardingTargetFields(1, 0)); assertNotNull(semantics.getForwardingTargetFields(0, 0)); assertNotNull(semantics.getForwardingTargetFields(1, 1)); assertEquals(1, semantics.getForwardingTargetFields(0, 0).size()); assertEquals(1, semantics.getForwardingTargetFields(1, 1).size()); assertTrue(semantics.getForwardingTargetFields(0, 0).contains(1)); assertTrue(semantics.getForwardingTargetFields(1, 1).contains(2)); assertEquals(0, semantics.getForwardingTargetFields(0, 1).size()); assertEquals(0, semantics.getForwardingTargetFields(1, 0).size()); }
/** * Computes the intersection between the edge set and the given edge set. For all matching pairs, * only one edge will be in the resulting data set. * * @param edges edges to compute intersection with * @return edge set containing one edge for all matching pairs of the same edge */ private DataSet<Edge<K, EV>> getDistinctEdgeIntersection(DataSet<Edge<K, EV>> edges) { return this.getEdges() .join(edges) .where(0, 1, 2) .equalTo(0, 1, 2) .with(new JoinFunction<Edge<K, EV>, Edge<K, EV>, Edge<K, EV>>() { @Override public Edge<K, EV> join(Edge<K, EV> first, Edge<K, EV> second) throws Exception { return first; } }).withForwardedFieldsFirst("*").name("Intersect edges") .distinct() .name("Edges"); }
@Test(expected = SemanticProperties.InvalidSemanticAnnotationException.class) public void testBinaryForwardedOverwritingInLine3() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(3L, 4L)); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input2 = env.fromElements(new Tuple2<Long, Long>(3L, 2L)); input1.join(input2).where(0).equalTo(0).with(new ForwardedBothAnnotationJoin<Long, Long, Long, Long>()) .withForwardedFieldsFirst("0->1;"); }
@Test(expected = SemanticProperties.InvalidSemanticAnnotationException.class) public void testBinaryForwardedOverwritingInLine5() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); @SuppressWarnings("unchecked") DataSet<Tuple3<Long, Long, Long>> input1 = env.fromElements(new Tuple3<Long, Long, Long>(3L, 4L, 5L)); @SuppressWarnings("unchecked") DataSet<Tuple3<Long, Long, Long>> input2 = env.fromElements(new Tuple3<Long, Long, Long>(3L, 2L, 1L)); input1.join(input2).where(0).equalTo(0).with(new AllForwardedExceptJoin<Long>()) .withForwardedFieldsFirst("0->1;"); }
@Test(expected = SemanticProperties.InvalidSemanticAnnotationException.class) public void testBinaryForwardedOverwritingInLine1() { ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input1 = env.fromElements(new Tuple2<Long, Long>(3L, 4L)); @SuppressWarnings("unchecked") DataSet<Tuple2<Long, Long>> input2 = env.fromElements(new Tuple2<Long, Long>(3L, 2L)); input1.join(input2).where(0).equalTo(0).with(new ForwardedFirstAnnotationJoin<Long>()) .withForwardedFieldsFirst("0->1"); }