private <VVWithDegree> void configureUpdateFunction(CoGroupOperator<?, ?, Vertex<K, VVWithDegree>> updates) { // configure coGroup update function with name and broadcast variables updates = updates.name("Vertex State Updates"); if (this.configuration != null) { for (Tuple2<String, DataSet<?>> e : this.configuration.getGatherBcastVars()) { updates = updates.withBroadcastSet(e.f1, e.f0); } } // let the operator know that we preserve the key field updates.withForwardedFieldsFirst("0").withForwardedFieldsSecond("0"); } }
return vertices.coGroup(edges).where(0).equalTo(1) .with(new ApplyCoGroupFunction<>(edgesFunction)) .name("GroupReduce on in-edges").returns(typeInfo); case OUT: return vertices.coGroup(edges).where(0).equalTo(0) .with(new ApplyCoGroupFunction<>(edgesFunction)) .name("GroupReduce on out-edges").returns(typeInfo); case ALL: return vertices.coGroup(edges.flatMap(new EmitOneEdgePerNode<>()) .name("Emit edge")) .where(0).equalTo(0).with(new ApplyCoGroupFunctionOnAllEdges<>(edgesFunction)) .name("GroupReduce on in- and out-edges").returns(typeInfo); default: throw new IllegalArgumentException("Illegal edge direction");
DataSet<Vertex<K, VV>> solutionSetDelta = superstepComputation.flatMap( new ProjectNewVertexValue<>()).returns(vertexType); DataSet<Tuple2<K, Either<NullValue, Message>>> allMessages = superstepComputation.flatMap( new ProjectMessages<>()).returns(workSetTypeInfo); superstepComputation = superstepComputation.name("Compute Function"); if (this.configuration != null) { for (Tuple2<String, DataSet<?>> e : this.configuration.getBcastVars()) { superstepComputation = superstepComputation.withBroadcastSet(e.f1, e.f0);
messages = messages.name("Messaging"); if (this.configuration != null) { for (Tuple2<String, DataSet<?>> e : this.configuration.getScatterBcastVars()) { messages = messages.withBroadcastSet(e.f1, e.f0); messages = messages.withBroadcastSet(numberOfVertices, "number of vertices");
.equalTo(1) .with(new Hubbiness<>()) .setParallelism(parallelism) .name("Hub") .groupBy(0) .reduce(new SumScore<>()) .setCombineHint(CombineHint.HASH) .equalTo(0) .with(new Authority<>()) .setParallelism(parallelism) .name("Authority") .groupBy(0) .reduce(new SumScore<>()) .setCombineHint(CombineHint.HASH)
/** * Return the out-degree of all vertices in the graph. * * @return A DataSet of {@code Tuple2<vertexId, outDegree>} */ public DataSet<Tuple2<K, LongValue>> outDegrees() { return vertices.coGroup(edges).where(0).equalTo(0).with(new CountNeighborsCoGroup<>()) .name("Out-degree"); }
case ALL: messages = buildScatterFunction(iteration, messageTypeInfo, 1, 0, numberOfVertices) .union(buildScatterFunction(iteration, messageTypeInfo, 0, 0, numberOfVertices)); break; default: updates = updates.withBroadcastSet(numberOfVertices, "number of vertices");
.where(new DummyTestKeySelector()).equalTo(new DummyTestKeySelector()) .with(new DummyTestCoGroupFunction2()) .withForwardedFieldsFirst("2;4->0") .withForwardedFieldsSecond("0->4;1;1->3"); SemanticProperties semProps = coGroupOp.getSemanticProperties();
.where(new Id<>()).equalTo(new Id<>()) .with(new LeftSide<>()) .flatMap(new MapVertexToPairWithGraphId()); .where(new Value1Of2<>()).equalTo(new Value1Of2<>()) .with(new CoGroupAssociateOldVerticesWithNewIds()) .union(vi.map(new MapVerticesAsTuplesWithNullId())); .where(new Value1Of2<>()).equalTo(new Value1Of2<>()) .with(new LeftSide<>()) .map(new Value0Of2<>()) .union(vi) .map(new MapFunctionAddGraphElementToGraph2<>(newGraphid));
.withPartitioner(partitioner) .with(new DummyCoGroupFunction<Tuple2<Long, Long>, Tuple3<Long, Long, Long>>()) .output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple3<Long, Long, Long>>>());
@Test public void testCoGroupWithTypeInformationTypeHint() throws Exception { final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); env.getConfig().disableSysoutLogging(); DataSet<Tuple3<Integer, Long, String>> ds1 = CollectionDataSets.getSmall3TupleDataSet(env); DataSet<Tuple3<Integer, Long, String>> ds2 = CollectionDataSets.getSmall3TupleDataSet(env); DataSet<Integer> resultDs = ds1 .coGroup(ds2) .where(0) .equalTo(0) .with(new CoGrouper<Tuple3<Integer, Long, String>, Tuple3<Integer, Long, String>, Integer>()) .returns(BasicTypeInfo.INT_TYPE_INFO); List<Integer> result = resultDs.collect(); String expectedResult = "2\n" + "3\n" + "1\n"; compareResultAsText(result, expectedResult); }
.sortFirstGroup(5, Order.DESCENDING) .sortSecondGroup(1, Order.DESCENDING).sortSecondGroup(4, Order.ASCENDING) .with(new IdentityCoGrouper<Tuple7<Long, Long, Long, Long, Long, Long, Long>>()).name("CoGroup") .output(new DiscardingOutputFormat<Tuple7<Long, Long, Long, Long, Long, Long, Long>>()).name("Sink");
/** * Checks that the edge set input contains valid vertex Ids, i.e. that they * also exist in the vertex input set. * * @return a boolean stating whether a graph is valid * with respect to its vertex ids. */ @Override public boolean validate(Graph<K, VV, EV> graph) throws Exception { DataSet<Tuple1<K>> edgeIds = graph.getEdges() .flatMap(new MapEdgeIds<>()).distinct(); DataSet<K> invalidIds = graph.getVertices().coGroup(edgeIds).where(0) .equalTo(0).with(new GroupInvalidIds<>()).first(1); return invalidIds.map(new KToTupleMap<>()).count() == 0; }
.withParameters(cfg) .output(new DiscardingOutputFormat<Tuple2<Tuple2<Long, Long>, Tuple2<Long, Long>>>());
@Test public void testCoGroupWithBroadcastSet() throws Exception { /* * Reduce with broadcast set */ final ExecutionEnvironment env = ExecutionEnvironment.getExecutionEnvironment(); DataSet<Integer> intDs = CollectionDataSets.getIntegerDataSet(env); DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds = CollectionDataSets.get5TupleDataSet(env); DataSet<Tuple5<Integer, Long, Integer, String, Long>> ds2 = CollectionDataSets.get5TupleDataSet(env); DataSet<Tuple3<Integer, Integer, Integer>> coGroupDs = ds.coGroup(ds2).where(0).equalTo(0).with(new Tuple5CoGroupBC()).withBroadcastSet(intDs, "ints"); List<Tuple3<Integer, Integer, Integer>> result = coGroupDs.collect(); String expected = "1,0,55\n" + "2,6,55\n" + "3,24,55\n" + "4,60,55\n" + "5,120,55\n"; compareResultAsTuples(result, expected); }
}).withForwardedFieldsFirst("0").withForwardedFieldsSecond("0");
.equalTo(0) .with(new SendScore<>()) .setParallelism(parallelism) .name("Send score") .groupBy(0) .reduce(new SumScore<>()) .setCombineHint(CombineHint.HASH)
/** * Return the in-degree of all vertices in the graph. * * @return A DataSet of {@code Tuple2<vertexId, inDegree>} */ public DataSet<Tuple2<K, LongValue>> inDegrees() { return vertices.coGroup(edges).where(0).equalTo(1).with(new CountNeighborsCoGroup<>()) .name("In-degree"); }
messages = messages.name("Messaging"); messages = messages.withBroadcastSet(e.f1, e.f0); messages = messages.withBroadcastSet(numberOfVertices, "number of vertices");