@Override public void close() throws Exception { this.scatterFunction.postSuperstep(); }
@Override public void close() throws Exception { this.gatherFunction.postSuperstep(); }
/** * Creates a new scatter-gather iteration operator for graphs where the edges are associated with a value (such as * a weight or distance). * * @param edgesWithValue The data set containing edges. * @param sf The function that turns changed vertex states into messages along the edges. * @param gf The function that updates the state of the vertices from the incoming messages. * * @param <K> The type of the vertex key (the vertex identifier). * @param <VV> The type of the vertex value (the state of the vertex). * @param <Message> The type of the message sent between vertices along the edges. * @param <EV> The type of the values that are associated with the edges. * * @return An in stance of the scatter-gather graph computation operator. */ public static <K, VV, Message, EV> ScatterGatherIteration<K, VV, Message, EV> withEdges( DataSet<Edge<K, EV>> edgesWithValue, ScatterFunction<K, VV, Message, EV> sf, GatherFunction<K, VV, Message> gf, int maximumNumberOfIterations) { return new ScatterGatherIteration<>(sf, gf, edgesWithValue, maximumNumberOfIterations); }
if (this.configuration != null && this.configuration.isOptNumVertices()) { try { numberOfVertices = GraphUtils.count(this.initialVertices); scatterFunction.setDirection(this.configuration.getDirection()); } else { scatterFunction.setDirection(EdgeDirection.OUT); EdgeDirection messagingDirection = scatterFunction.getDirection(); if (this.configuration != null && this.configuration.isOptDegrees()) { return createResultVerticesWithDegrees(graph, messagingDirection, messageTypeInfo, numberOfVertices); } else { return createResultSimpleVertex(messagingDirection, messageTypeInfo, numberOfVertices);
@Override public void open(Configuration parameters) throws Exception { if (getRuntimeContext().hasBroadcastVariable("number of vertices")) { Collection<LongValue> numberOfVertices = getRuntimeContext().getBroadcastVariable("number of vertices"); this.gatherFunction.setNumberOfVertices(numberOfVertices.iterator().next().getValue()); } if (getIterationRuntimeContext().getSuperstepNumber() == 1) { this.gatherFunction.init(getIterationRuntimeContext()); } this.gatherFunction.preSuperstep(); }
@Override public void open(Configuration parameters) throws Exception { if (getRuntimeContext().hasBroadcastVariable("number of vertices")) { Collection<LongValue> numberOfVertices = getRuntimeContext().getBroadcastVariable("number of vertices"); this.scatterFunction.setNumberOfVertices(numberOfVertices.iterator().next().getValue()); } if (getIterationRuntimeContext().getSuperstepNumber() == 1) { this.scatterFunction.init(getIterationRuntimeContext()); } this.scatterFunction.preSuperstep(); }
@Override public void coGroup(Iterable<Tuple2<K, Message>> messages, Iterable<Vertex<K, Tuple3<VV, LongValue, LongValue>>> vertex, Collector<Vertex<K, Tuple3<VV, LongValue, LongValue>>> out) throws Exception { final Iterator<Vertex<K, Tuple3<VV, LongValue, LongValue>>> vertexIter = vertex.iterator(); if (vertexIter.hasNext()) { Vertex<K, Tuple3<VV, LongValue, LongValue>> vertexWithDegrees = vertexIter.next(); @SuppressWarnings("unchecked") Iterator<Tuple2<?, Message>> downcastIter = (Iterator<Tuple2<?, Message>>) (Iterator<?>) messages.iterator(); messageIter.setSource(downcastIter); gatherFunction.setInDegree(vertexWithDegrees.f1.f1.getValue()); gatherFunction.setOutDegree(vertexWithDegrees.f1.f2.getValue()); gatherFunction.setOutputWithDegrees(vertexWithDegrees, out); gatherFunction.updateVertexFromScatterGatherIteration(vertexWithDegrees, messageIter); } else { final Iterator<Tuple2<K, Message>> messageIter = messages.iterator(); if (messageIter.hasNext()) { String message = "Target vertex does not exist!."; try { Tuple2<K, Message> next = messageIter.next(); message = "Target vertex '" + next.f0 + "' does not exist!."; } catch (Throwable ignored) {} throw new Exception(message); } else { throw new Exception(); } } } }
@Override public void coGroup(Iterable<Edge<K, EV>> edges, Iterable<Vertex<K, Tuple3<VV, LongValue, LongValue>>> state, Collector<Tuple2<K, Message>> out) throws Exception { final Iterator<Vertex<K, Tuple3<VV, LongValue, LongValue>>> stateIter = state.iterator(); if (stateIter.hasNext()) { Vertex<K, Tuple3<VV, LongValue, LongValue>> vertexWithDegrees = stateIter.next(); nextVertex.f0 = vertexWithDegrees.f0; nextVertex.f1 = vertexWithDegrees.f1.f0; scatterFunction.setInDegree(vertexWithDegrees.f1.f1.getValue()); scatterFunction.setOutDegree(vertexWithDegrees.f1.f2.getValue()); scatterFunction.set(edges.iterator(), out, vertexWithDegrees.getId()); scatterFunction.sendMessages(nextVertex); } } }
/** * Helper method which sets up an iteration with the given vertex value(either simple or with degrees). * * @param iteration */ private void setUpIteration(DeltaIteration<?, ?> iteration) { // set up the iteration operator if (this.configuration != null) { iteration.name(this.configuration.getName("Scatter-gather iteration (" + gatherFunction + " | " + scatterFunction + ")")); iteration.parallelism(this.configuration.getParallelism()); iteration.setSolutionSetUnManaged(this.configuration.isSolutionSetUnmanagedMemory()); // register all aggregators for (Map.Entry<String, Aggregator<?>> entry : this.configuration.getAggregators().entrySet()) { iteration.registerAggregator(entry.getKey(), entry.getValue()); } } else { // no configuration provided; set default name iteration.name("Scatter-gather iteration (" + gatherFunction + " | " + scatterFunction + ")"); } }
@Override public void coGroup(Iterable<Tuple2<K, Message>> messages, Iterable<Vertex<K, VV>> vertex, Collector<Vertex<K, VV>> out) throws Exception { final Iterator<Vertex<K, VV>> vertexIter = vertex.iterator(); if (vertexIter.hasNext()) { Vertex<K, VV> vertexState = vertexIter.next(); @SuppressWarnings("unchecked") Iterator<Tuple2<?, Message>> downcastIter = (Iterator<Tuple2<?, Message>>) (Iterator<?>) messages.iterator(); messageIter.setSource(downcastIter); gatherFunction.setOutput(vertexState, out); gatherFunction.updateVertex(vertexState, messageIter); } else { final Iterator<Tuple2<K, Message>> messageIter = messages.iterator(); if (messageIter.hasNext()) { String message = "Target vertex does not exist!."; try { Tuple2<K, Message> next = messageIter.next(); message = "Target vertex '" + next.f0 + "' does not exist!."; } catch (Throwable ignored) {} throw new Exception(message); } else { throw new Exception(); } } } }
@Override public void coGroup(Iterable<Edge<K, EV>> edges, Iterable<Vertex<K, VV>> state, Collector<Tuple2<K, Message>> out) throws Exception { final Iterator<Vertex<K, VV>> stateIter = state.iterator(); if (stateIter.hasNext()) { Vertex<K, VV> newVertexState = stateIter.next(); scatterFunction.set(edges.iterator(), out, newVertexState.getId()); scatterFunction.sendMessages(newVertexState); } } }
/** * Runs a ScatterGather iteration on the graph with configuration options. * * @param scatterFunction the scatter function * @param gatherFunction the gather function * @param maximumNumberOfIterations maximum number of iterations to perform * @param parameters the iteration configuration parameters * * @return the updated Graph after the scatter-gather iteration has converged or * after maximumNumberOfIterations. */ public <M> Graph<K, VV, EV> runScatterGatherIteration( ScatterFunction<K, VV, M, EV> scatterFunction, org.apache.flink.graph.spargel.GatherFunction<K, VV, M> gatherFunction, int maximumNumberOfIterations, ScatterGatherConfiguration parameters) { ScatterGatherIteration<K, VV, M, EV> iteration = ScatterGatherIteration.withEdges( edges, scatterFunction, gatherFunction, maximumNumberOfIterations); iteration.configure(parameters); DataSet<Vertex<K, VV>> newVertices = this.getVertices().runOperation(iteration); return new Graph<>(newVertices, this.edges, this.context); }
/** * Gets an {@link java.lang.Iterable} with all edges. This method is mutually exclusive with * {@link #sendMessageToAllNeighbors(Object)} and may be called only once. * * <p>If the {@link EdgeDirection} is OUT (default), then this iterator contains outgoing edges. * * <p>If the {@link EdgeDirection} is IN, then this iterator contains incoming edges. * * <p>If the {@link EdgeDirection} is ALL, then this iterator contains both outgoing and incoming edges. * * @return An iterator with all edges. */ @SuppressWarnings("unchecked") public Iterable<Edge<K, EV>> getEdges() { if (edgesUsed) { throw new IllegalStateException("Can use either 'getEdges()' or 'sendMessageToAllNeighbors()' exactly once."); } edgesUsed = true; this.edgeIterator.set((Iterator<Edge<K, EV>>) edges); return this.edgeIterator; }
void init(IterationRuntimeContext context) { this.runtimeContext = context; this.outValue = new Tuple2<>(); this.edgeIterator = new EdgesIterator<>(); }
/** * In order to hide the Tuple3(actualValue, inDegree, OutDegree) vertex value from the user, * another function will be called from {@link org.apache.flink.graph.spargel.ScatterGatherIteration}. * * <p>This function will retrieve the vertex from the vertexState and will set its degrees, afterwards calling * the regular updateVertex function. * * @param vertexState * @param inMessages * @throws Exception */ @SuppressWarnings("unchecked") <VertexWithDegree> void updateVertexFromScatterGatherIteration(Vertex<K, VertexWithDegree> vertexState, MessageIterator<Message> inMessages) throws Exception { Vertex<K, VV> vertex = new Vertex<>(vertexState.f0, ((Tuple3<VV, Long, Long>) vertexState.getValue()).f0); updateVertex(vertex, inMessages); } }
private <VVWithDegree> void configureUpdateFunction(CoGroupOperator<?, ?, Vertex<K, VVWithDegree>> updates) { // configure coGroup update function with name and broadcast variables updates = updates.name("Vertex State Updates"); if (this.configuration != null) { for (Tuple2<String, DataSet<?>> e : this.configuration.getGatherBcastVars()) { updates = updates.withBroadcastSet(e.f1, e.f0); } } // let the operator know that we preserve the key field updates.withForwardedFieldsFirst("0").withForwardedFieldsSecond("0"); } }
private ScatterGatherIteration(ScatterFunction<K, VV, Message, EV> sf, GatherFunction<K, VV, Message> gf, DataSet<Edge<K, EV>> edgesWithValue, int maximumNumberOfIterations) { Preconditions.checkNotNull(sf); Preconditions.checkNotNull(gf); Preconditions.checkNotNull(edgesWithValue); Preconditions.checkArgument(maximumNumberOfIterations > 0, "The maximum number of iterations must be at least one."); this.scatterFunction = sf; this.gatherFunction = gf; this.edgesWithValue = edgesWithValue; this.maximumNumberOfIterations = maximumNumberOfIterations; this.messageType = getMessageType(sf); }
/** * Sets the new value of this vertex. Setting a new value triggers the sending of outgoing messages from this vertex. * * <p>This should be called at most once per updateVertex. * * @param newValue The new vertex value. */ public void setNewVertexValue(VV newValue) { if (setNewVertexValueCalled) { throw new IllegalStateException("setNewVertexValue should only be called at most once per updateVertex"); } setNewVertexValueCalled = true; if (isOptDegrees()) { outValWithDegrees.f1.f0 = newValue; outWithDegrees.collect(outValWithDegrees); } else { outVal.setValue(newValue); out.collect(outVal); } }
@Override public DataSet<Vertex<K, Double>> run(Graph<K, Double, Double> network) throws Exception { DataSet<Tuple2<K, LongValue>> vertexOutDegrees = network.outDegrees(); Graph<K, Double, Double> networkWithWeights = network .joinWithEdgesOnSource(vertexOutDegrees, new InitWeights()); ScatterGatherConfiguration parameters = new ScatterGatherConfiguration(); parameters.setOptNumVertices(true); return networkWithWeights.runScatterGatherIteration(new RankMessenger<>(), new VertexRankUpdater<>(beta), maxIterations, parameters) .getVertices(); }
@Override public void updateVertex(Vertex<Long, Double> vertex, MessageIterator<Double> inMessages) throws Exception { if (inMessages.hasNext()) { Long outDegree = getOutDegree() - 1; // check if the vertex has another SP-Edge if (outDegree <= 0) { // set own value to infinity setNewVertexValue(Double.MAX_VALUE); } } } }