/** * Filter tuples that are representing a single vertex. They are used to update the source and target vertex * identifiers at the edges. * * @param <K> vertex identifier type * @param <VV> vertex value type */ @SuppressWarnings("serial") @FunctionAnnotation.ForwardedFields("*->*") private static final class VertexGroupItemToRepresentativeFilter<K, VV> implements FilterFunction<VertexGroupItem<K, VV>> { @Override public boolean filter(VertexGroupItem<K, VV> vertexGroupItem) throws Exception { return vertexGroupItem.getVertexGroupCount().equals(0L); } }
@ForwardedFields("1") private static class Map34 implements MapFunction<Tuple2<Long, Long>, Tuple2<Long, Long>> { private Tuple2<Long, Long> t; @Override public Tuple2<Long, Long> map(Tuple2<Long, Long> value) throws Exception { if (value != new Object()) { return value; } else if (value.f0 == 1L && value.f1 == 2L) { t = value; t.f0 = 23L; return t; } return new Tuple2<Long, Long>(value.f0, value.f1); } }
/** * Combines the vertex degree count. * * @param <K> ID type */ @ForwardedFields("0") public static class DegreeCount<K> implements ReduceFunction<Vertex<K, LongValue>> { @Override public Vertex<K, LongValue> reduce(Vertex<K, LongValue> left, Vertex<K, LongValue> right) throws Exception { LongValue count = left.f1; count.setValue(count.getValue() + right.f1.getValue()); return left; } }
/** * Sum the PageRank score over all vertices. The vertex ID must be ignored * but is retained rather than adding another operator. * * @param <T> ID type */ @ForwardedFields("0") private static class SumVertexScores<T> implements ReduceFunction<Tuple2<T, DoubleValue>> { @Override public Tuple2<T, DoubleValue> reduce(Tuple2<T, DoubleValue> first, Tuple2<T, DoubleValue> second) throws Exception { first.f1.setValue(first.f1.getValue() + second.f1.getValue()); return first; } }
/** * Filter tuples that are representing a vertex group. They are used to create new summarized vertices and have a * group count greater than zero. * * @param <K> vertex identifier type * @param <VV> vertex value type */ @SuppressWarnings("serial") @FunctionAnnotation.ForwardedFields("*->*") private static final class VertexGroupItemToSummarizedVertexFilter<K, VV> implements FilterFunction<VertexGroupItem<K, VV>> { @Override public boolean filter(VertexGroupItem<K, VV> vertexGroupItem) throws Exception { return !vertexGroupItem.getVertexGroupCount().equals(0L); } }
/** * Sum vertices' hub and authority scores. * * @param <T> ID type */ @ForwardedFields("0") private static class SumScores<T> implements ReduceFunction<Tuple3<T, DoubleValue, DoubleValue>> { @Override public Tuple3<T, DoubleValue, DoubleValue> reduce(Tuple3<T, DoubleValue, DoubleValue> left, Tuple3<T, DoubleValue, DoubleValue> right) throws Exception { left.f1.setValue(left.f1.getValue() + right.f1.getValue()); left.f2.setValue(left.f2.getValue() + right.f2.getValue()); return left; } }
/** * Map the Tuple result to the return type. * * @param <T> ID type */ @ForwardedFields("0->vertexId0; 1->pageRankScore") private static class TranslateResult<T> implements MapFunction<Tuple2<T, DoubleValue>, Result<T>> { private Result<T> output = new Result<>(); @Override public Result<T> map(Tuple2<T, DoubleValue> value) throws Exception { output.setVertexId0(value.f0); output.setPageRankScore(value.f1); return output; } }
/** * Emits the target vertex ID along with an initial count. * * @param <K> ID type * @param <EV> edge value type */ @ForwardedFields("1->0") public static class MapEdgeToTargetId<K, EV> implements MapFunction<Edge<K, EV>, Vertex<K, LongValue>> { private Vertex<K, LongValue> output = new Vertex<>(null, new LongValue(1)); @Override public Vertex<K, LongValue> map(Edge<K, EV> value) throws Exception { output.f0 = value.f1; return output; } }
/** * Emits the source vertex ID along with an initial count. * * @param <K> ID type * @param <EV> edge value type */ @ForwardedFields("0") public static class MapEdgeToSourceId<K, EV> implements MapFunction<Edge<K, EV>, Vertex<K, LongValue>> { private Vertex<K, LongValue> output = new Vertex<>(null, new LongValue(1)); @Override public Vertex<K, LongValue> map(Edge<K, EV> value) throws Exception { output.f0 = value.f0; return output; } }
@ForwardedFields("*->f0") private static class LinkVertexToCenter implements FlatMapFunction<LongValue, Edge<LongValue, NullValue>> { private LongValue center = new LongValue(0); private Edge<LongValue, NullValue> centerToLeaf = new Edge<>(center, null, NullValue.getInstance()); private Edge<LongValue, NullValue> leafToCenter = new Edge<>(null, center, NullValue.getInstance()); @Override public void flatMap(LongValue leaf, Collector<Edge<LongValue, NullValue>> out) throws Exception { centerToLeaf.f1 = leaf; out.collect(centerToLeaf); leafToCenter.f0 = leaf; out.collect(leafToCenter); } } }
/** * Sum vertices' scores. * * @param <T> ID type */ @ForwardedFields("0") protected static final class SumScore<T> implements ReduceFunction<Tuple2<T, DoubleValue>> { @Override public Tuple2<T, DoubleValue> reduce(Tuple2<T, DoubleValue> left, Tuple2<T, DoubleValue> right) throws Exception { left.f1.setValue(left.f1.getValue() + right.f1.getValue()); return left; } } }
@ForwardedFields("*") private static class Map23 implements MapFunction<Tuple1<Integer>, Tuple1<Integer>> { @Override public Tuple1<Integer> map(Tuple1<Integer> value) throws Exception { if (value.f0.equals(23)) { return new Tuple1<Integer>(value.<Integer> getField(0)); } else if (value.f0.equals(22)) { Tuple1<Integer> inputContainer = new Tuple1<Integer>(); inputContainer.f0 = value.f0; return new Tuple1<Integer>(inputContainer.<Integer> getField(0)); } else { return value; } } }
/** * Sums the triangle count for each vertex ID. * * @param <T> ID type */ @ForwardedFields("0") private static class CountTriangles<T> implements ReduceFunction<Tuple2<T, LongValue>> { @Override public Tuple2<T, LongValue> reduce(Tuple2<T, LongValue> left, Tuple2<T, LongValue> right) throws Exception { left.f1.setValue(left.f1.getValue() + right.f1.getValue()); return left; } }
/** * Sums the triangle count for each vertex ID. * * @param <T> ID type */ @ForwardedFields("0") private static class CountTriangles<T> implements ReduceFunction<Tuple2<T, LongValue>> { @Override public Tuple2<T, LongValue> reduce(Tuple2<T, LongValue> left, Tuple2<T, LongValue> right) throws Exception { left.f1.setValue(left.f1.getValue() + right.f1.getValue()); return left; } }
/** * Create a Tuple3 DataSet from an Edge DataSet. * * @param <K> edge ID type * @param <EV> edge value type */ @ForwardedFields("f0; f1; f2") public class EdgeToTuple3Map<K, EV> implements MapFunction<Edge<K, EV>, Tuple3<K, K, EV>> { private static final long serialVersionUID = 1L; @Override public Tuple3<K, K, EV> map(Edge<K, EV> edge) { return edge; } }
@ForwardedFields("0") private static class BuildOutgoingEdgeList implements GroupReduceFunction<Tuple2<Long, Long>, Tuple2<Long, Long[]>> { private final ArrayList<Long> neighbors = new ArrayList<Long>(); @Override public void reduce(Iterable<Tuple2<Long, Long>> values, Collector<Tuple2<Long, Long[]>> out) { neighbors.clear(); Long id = 0L; for (Tuple2<Long, Long> n : values) { id = n.f0; neighbors.add(n.f1); } out.collect(new Tuple2<Long, Long[]>(id, neighbors.toArray(new Long[neighbors.size()]))); } }
/** * Create a Tuple2 DataSet from a Vertex DataSet. * * @param <K> vertex ID type * @param <VV> vertex value type */ @ForwardedFields("f0; f1") public class VertexToTuple2Map<K, VV> implements MapFunction<Vertex<K, VV>, Tuple2<K, VV>> { private static final long serialVersionUID = 1L; @Override public Tuple2<K, VV> map(Vertex<K, VV> vertex) { return vertex; } }
/** * Mapper that removes keys. * @param <T> type of values * @param <K> type of keys */ @Internal @ForwardedFields("1->*") public final class KeyRemovingMapper<T, K> extends RichMapFunction<Tuple2<K, T>, T> { private static final long serialVersionUID = 1L; @Override public T map(Tuple2<K, T> value) { return value.f1; } }
@ForwardedFields("field2") private static class Map40 implements MapFunction<MyPojo, MyPojo> { @Override public MyPojo map(MyPojo value) throws Exception { return recursiveFunction(value); } private MyPojo recursiveFunction(MyPojo value) { if (value.field.equals("xyz")) { value.field = value.field + "x"; return recursiveFunction(value); } return value; } }
@ForwardedFields("0") private static class DocumentReducer implements GroupReduceFunction<Tuple2<Integer, String>, Document> { @Override public void reduce(Iterable<Tuple2<Integer, String>> values, Collector<Document> out) throws Exception { Iterator<Tuple2<Integer, String>> it = values.iterator(); Tuple2<Integer, String> first = it.next(); Integer docId = first.f0; StringBuilder builder = new StringBuilder(first.f1); while (it.hasNext()) { builder.append("-").append(it.next().f1); } out.collect(new Document(docId, false, false, "", builder.toString())); } }