private void addSlidingWindowSingleStage(Planner p, SlidingWindowDef wDef) { PlannerVertex pv = p.addVertex(this, p.uniqueVertexName(name()), localParallelism(), aggregateToSlidingWindowP( keyFns, nCopies(keyFns.size(), (DistributedToLongFunction<JetEvent>) JetEvent::timestamp), TimestampKind.EVENT, wDef.toSlidingWindowPolicy(), aggrOp, mapToOutputFn )); p.addEdges(this, pv.v, (e, ord) -> e.distributed().partitioned(keyFns.get(ord))); }
private void addSessionWindow(Planner p, SessionWindowDef wDef) { PlannerVertex pv = p.addVertex(this, p.uniqueVertexName(name()), localParallelism(), aggregateToSessionWindowP( wDef.sessionTimeout(), nCopies(keyFns.size(), (DistributedToLongFunction<JetEvent>) JetEvent::timestamp), keyFns, aggrOp, mapToOutputFn )); p.addEdges(this, pv.v, (e, ord) -> e.distributed().partitioned(keyFns.get(ord))); } }
private void addToDagSingleStage(Planner p) { PlannerVertex pv = p.addVertex(this, p.uniqueVertexName(name()), localParallelism(), aggregateByKeyP(groupKeyFns, aggrOp, mapToOutputFn)); p.addEdges(this, pv.v, (e, ord) -> e.distributed().partitioned(groupKeyFns.get(ord))); }
@Override public void addToDag(Planner p) { PlannerVertex pv = p.addVertex(this, p.uniqueVertexName(name()), localParallelism(), processorSupplier); p.addEdges(this, pv.v, e -> e.partitioned(partitionKeyFn).distributed()); } }
private void addSlidingWindowSingleStage(Planner p, SlidingWindowDef wDef) { PlannerVertex pv = p.addVertex(this, p.uniqueVertexName(name()), 1, aggregateToSlidingWindowP( nCopies(aggrOp.arity(), constantKey()), nCopies(aggrOp.arity(), (DistributedToLongFunction<JetEvent>) JetEvent::timestamp), TimestampKind.EVENT, wDef.toSlidingWindowPolicy(), aggrOp, mapToOutputFn.toKeyedWindowResultFn() )); p.addEdges(this, pv.v, edge -> edge.distributed().allToOne()); }
private void addToDagSingleStage(Planner p) { PlannerVertex pv = p.addVertex(this, p.uniqueVertexName(name()), 1, aggregateP(aggrOp)); p.addEdges(this, pv.v, edge -> edge.distributed().allToOne()); }
private void addSessionWindow(Planner p, SessionWindowDef wDef) { PlannerVertex pv = p.addVertex(this, p.uniqueVertexName(name()), localParallelism(), aggregateToSessionWindowP( wDef.sessionTimeout(), nCopies(aggrOp.arity(), (DistributedToLongFunction<JetEvent>) JetEvent::timestamp), nCopies(aggrOp.arity(), constantKey()), aggrOp, mapToOutputFn.toKeyedWindowResultFn())); p.addEdges(this, pv.v, edge -> edge.distributed().allToOne()); } }
@Override public void addToDag(Planner p) { PlannerVertex pv = p.addVertex(this, p.uniqueVertexName(name()), localParallelism(), rollingAggregateP(keyFn, aggrOp, mapToOutputFn)); p.addEdges(this, pv.v, edge -> edge.partitioned(keyFn).distributed()); } }
private void addSlidingWindowTwoStage(Planner p, SlidingWindowDef wDef) { String vertexName = p.uniqueVertexName(name()); SlidingWindowPolicy winPolicy = wDef.toSlidingWindowPolicy(); Vertex v1 = p.dag.newVertex(vertexName + FIRST_STAGE_VERTEX_NAME_SUFFIX, accumulateByFrameP( keyFns, nCopies(keyFns.size(), (DistributedToLongFunction<JetEvent>) JetEvent::timestamp), TimestampKind.EVENT, winPolicy, aggrOp)); v1.localParallelism(localParallelism()); PlannerVertex pv2 = p.addVertex(this, vertexName, localParallelism(), combineToSlidingWindowP(winPolicy, aggrOp, mapToOutputFn)); p.addEdges(this, v1, (e, ord) -> e.partitioned(keyFns.get(ord), HASH_CODE)); p.dag.edge(between(v1, pv2.v).distributed().partitioned(entryKey())); }
private void addToDagTwoStage(Planner p) { List<DistributedFunction<?, ? extends K>> groupKeyFns = this.groupKeyFns; String vertexName = p.uniqueVertexName(this.name()); Vertex v1 = p.dag.newVertex(vertexName + FIRST_STAGE_VERTEX_NAME_SUFFIX, accumulateByKeyP(groupKeyFns, aggrOp)) .localParallelism(localParallelism()); PlannerVertex pv2 = p.addVertex(this, vertexName, localParallelism(), combineByKeyP(aggrOp, mapToOutputFn)); p.addEdges(this, v1, (e, ord) -> e.partitioned(groupKeyFns.get(ord), HASH_CODE)); p.dag.edge(between(v1, pv2.v).distributed().partitioned(entryKey())); } }
p.dag.edge(from(fromPv.v, fromPv.nextAvailableOrdinal()) .to(collector, 0) .distributed().broadcast()); p.dag.edge(from(collector, 0) .to(joiner, collectorOrdinal)
private void addSlidingWindowTwoStage(Planner p, SlidingWindowDef wDef) { String vertexName = p.uniqueVertexName(name()); SlidingWindowPolicy winPolicy = wDef.toSlidingWindowPolicy(); Vertex v1 = p.dag.newVertex(vertexName + FIRST_STAGE_VERTEX_NAME_SUFFIX, accumulateByFrameP( nCopies(aggrOp.arity(), constantKey()), nCopies(aggrOp.arity(), (DistributedToLongFunction<JetEvent>) JetEvent::timestamp), TimestampKind.EVENT, winPolicy, aggrOp )); v1.localParallelism(localParallelism()); PlannerVertex pv2 = p.addVertex(this, vertexName, 1, combineToSlidingWindowP(winPolicy, aggrOp, mapToOutputFn.toKeyedWindowResultFn())); p.addEdges(this, v1); p.dag.edge(between(v1, pv2.v).distributed().allToOne()); }
private void addToDagTwoStage(Planner p) { String vertexName = p.uniqueVertexName(name()); Vertex v1 = p.dag.newVertex(vertexName + FIRST_STAGE_VERTEX_NAME_SUFFIX, accumulateP(aggrOp)) .localParallelism(localParallelism()); PlannerVertex pv2 = p.addVertex(this, vertexName, 1, combineP(aggrOp)); p.addEdges(this, v1); p.dag.edge(between(v1, pv2.v).distributed().allToOne()); } }
.edge(between(slidingStage1, slidingStage2) .partitioned(entryKey(), HASH_CODE) .distributed()) .edge(between(slidingStage2, formatOutput) .isolated())
@Override public void addToDag(Planner p) { String vertexName = p.uniqueVertexName(this.name()); Vertex v1 = p.dag.newVertex(vertexName + FIRST_STAGE_VERTEX_NAME_SUFFIX, distinctP(keyFn)) .localParallelism(localParallelism()); PlannerVertex pv2 = p.addVertex(this, vertexName, localParallelism(), distinctP(keyFn)); p.addEdges(this, v1, (e, ord) -> e.partitioned(keyFn, HASH_CODE)); p.dag.edge(between(v1, pv2.v).distributed().partitioned(keyFn)); }
.partitioned(wholeItem(), HASH_CODE)) .edge(between(accumulate, combine) .distributed() .partitioned(entryKey())) .edge(between(combine, sink));
.edge(between(docSource, docCount).distributed().broadcast()) .edge(from(docSource, 1).to(docLines)) .edge(from(docLines).to(tokenize, 1)) .edge(between(tokenize, tf).partitioned(wholeItem(), HASH_CODE)) .edge(between(docCount, tfidf).broadcast().priority(-1)) .edge(from(tf).to(tfidf, 1).distributed().partitioned(byWord, HASH_CODE)) .edge(between(tfidf, sink));
.edge(between(streamTrades, slidingWindow) .partitioned(Trade::getTicker, HASH_CODE) .distributed()) .edge(between(slidingWindow, formatOutput).isolated()) .edge(between(formatOutput, sink));
.distributed()) .edge(from(collectToMap).to(hashJoin, 0) .broadcast()