@Override public void addToDag(Planner p) { String vertexName = p.uniqueVertexName(this.name()); Vertex v1 = p.dag.newVertex(vertexName + FIRST_STAGE_VERTEX_NAME_SUFFIX, distinctP(keyFn)) .localParallelism(localParallelism()); PlannerVertex pv2 = p.addVertex(this, vertexName, localParallelism(), distinctP(keyFn)); p.addEdges(this, v1, (e, ord) -> e.partitioned(keyFn, HASH_CODE)); p.dag.edge(between(v1, pv2.v).distributed().partitioned(keyFn)); }
Vertex joiner = p.addVertex(this, namePrefix + "-joiner", localParallelism(), () -> new HashJoinP<>(keyFns, tags, mapToOutputBiFn, mapToOutputTriFn)).v; p.dag.edge(from(primary.v, primary.nextAvailableOrdinal()).to(joiner, 0)); () -> new HashJoinCollectP(getKeyFn, projectFn)); collector.localParallelism(1); p.dag.edge(from(fromPv.v, fromPv.nextAvailableOrdinal()) .to(collector, 0) .distributed().broadcast()); p.dag.edge(from(collector, 0) .to(joiner, collectorOrdinal) .broadcast().priority(-1)); collectorOrdinal++;
public void addEdges(Transform transform, Vertex toVertex, BiConsumer<Edge, Integer> configureEdgeFn) { int destOrdinal = 0; for (Transform fromTransform : transform.upstream()) { PlannerVertex fromPv = xform2vertex.get(fromTransform); Edge edge = from(fromPv.v, fromPv.nextAvailableOrdinal()).to(toVertex, destOrdinal); dag.edge(edge); configureEdgeFn.accept(edge, destOrdinal); destOrdinal++; } }
private void addSessionWindow(Planner p, SessionWindowDef wDef) { PlannerVertex pv = p.addVertex(this, p.uniqueVertexName(name()), localParallelism(), aggregateToSessionWindowP( wDef.sessionTimeout(), nCopies(keyFns.size(), (DistributedToLongFunction<JetEvent>) JetEvent::timestamp), keyFns, aggrOp, mapToOutputFn )); p.addEdges(this, pv.v, (e, ord) -> e.distributed().partitioned(keyFns.get(ord))); } }
.edge(between(stopwordSource, tokenize).broadcast().priority(-1)) .edge(between(docSource, docCount).distributed().broadcast()) .edge(from(docSource, 1).to(docLines)) .edge(from(docLines).to(tokenize, 1)) .edge(between(tokenize, tf).partitioned(wholeItem(), HASH_CODE)) .edge(between(docCount, tfidf).broadcast().priority(-1)) .edge(from(tf).to(tfidf, 1).distributed().partitioned(byWord, HASH_CODE)) .edge(between(tfidf, sink));
.edge(between(streamTrades, slidingWindow) .partitioned(Trade::getTicker, HASH_CODE) .distributed()) .edge(between(slidingWindow, formatOutput).isolated()) .edge(between(formatOutput, sink));
private void addToDagTwoStage(Planner p) { String vertexName = p.uniqueVertexName(name()); Vertex v1 = p.dag.newVertex(vertexName + FIRST_STAGE_VERTEX_NAME_SUFFIX, accumulateP(aggrOp)) .localParallelism(localParallelism()); PlannerVertex pv2 = p.addVertex(this, vertexName, 1, combineP(aggrOp)); p.addEdges(this, v1); p.dag.edge(between(v1, pv2.v).distributed().allToOne()); } }
@Nonnull private static DAG buildDag(Map<String, Long> counts) { final Pattern delimiter = Pattern.compile("\\W+"); DAG dag = new DAG(); Vertex source = dag.newVertex("source", DocLinesP::new); Vertex tokenize = dag.newVertex("tokenize", flatMapP((String line) -> traverseArray(delimiter.split(line.toLowerCase())) .filter(word -> !word.isEmpty())) ); Vertex aggregate = dag.newVertex("aggregate", aggregateByKeyP(singletonList(wholeItem()), counting(), Util::entry)); Vertex sink = dag.newVertex("sink", () -> new MapSinkP(counts)); return dag.edge(between(source.localParallelism(1), tokenize)) .edge(between(tokenize, aggregate).partitioned(wholeItem(), HASH_CODE)) .edge(between(aggregate, sink)); }
private void rewriteDagWithSnapshotRestore(DAG dag, long snapshotId, String mapName) { IMap<Object, Object> snapshotMap = nodeEngine.getHazelcastInstance().getMap(mapName); snapshotId = SnapshotValidator.validateSnapshot(snapshotId, jobIdString(), snapshotMap); logger.info("State of " + jobIdString() + " will be restored from snapshot " + snapshotId + ", map=" + mapName); List<Vertex> originalVertices = new ArrayList<>(); dag.iterator().forEachRemaining(originalVertices::add); Map<String, Integer> vertexToOrdinal = new HashMap<>(); Vertex readSnapshotVertex = dag.newVertex(SNAPSHOT_VERTEX_PREFIX + "read", readMapP(mapName)); long finalSnapshotId = snapshotId; Vertex explodeVertex = dag.newVertex(SNAPSHOT_VERTEX_PREFIX + "explode", () -> new ExplodeSnapshotP(vertexToOrdinal, finalSnapshotId)); dag.edge(between(readSnapshotVertex, explodeVertex).isolated()); int index = 0; // add the edges for (Vertex userVertex : originalVertices) { vertexToOrdinal.put(userVertex.getName(), index); int destOrdinal = dag.getInboundEdges(userVertex.getName()).size(); dag.edge(new SnapshotRestoreEdge(explodeVertex, index, userVertex, destOrdinal)); index++; } }
private void addToDagSingleStage(Planner p) { PlannerVertex pv = p.addVertex(this, p.uniqueVertexName(name()), 1, aggregateP(aggrOp)); p.addEdges(this, pv.v, edge -> edge.distributed().allToOne()); }
public static void main(String[] args) throws Exception { System.setProperty("hazelcast.logging.type", "log4j"); Jet.newJetInstance(); JetInstance jet = Jet.newJetInstance(); try { IMapJet<Object, Object> map = jet.getMap("map"); range(0, COUNT).parallel().forEach(i -> map.put("key-" + i, i)); DAG dag = new DAG(); Vertex source = dag.newVertex("map-source", SourceProcessors.readMapP(map.getName())); Vertex sink = dag.newVertex("file-sink", new WriteFilePSupplier(OUTPUT_FOLDER)); dag.edge(between(source, sink)); jet.newJob(dag).join(); System.out.println("\nHazelcast IMap dumped to folder " + new File(OUTPUT_FOLDER).getAbsolutePath()); } finally { Jet.shutdownAll(); } } }
public static CompletableFuture<Void> copyMapUsingJob(JetInstance instance, int queueSize, String sourceMap, String targetMap) { DAG dag = new DAG(); Vertex source = dag.newVertex("readMap(" + sourceMap + ')', readMapP(sourceMap)); Vertex sink = dag.newVertex("writeMap(" + targetMap + ')', writeMapP(targetMap)); dag.edge(between(source, sink).setConfig(new EdgeConfig().setQueueSize(queueSize))); JobConfig jobConfig = new JobConfig() .setName("copy-" + sourceMap + "-to-" + targetMap); return instance.newJob(dag, jobConfig).getFuture(); } }
/** * Activates the {@link RoutingPolicy#PARTITIONED PARTITIONED} routing * policy and applies the {@link Partitioner#defaultPartitioner() default} * Hazelcast partitioning strategy. The strategy is applied to the result of * the {@code extractKeyFn} function. */ @Nonnull public <T> Edge partitioned(@Nonnull DistributedFunction<T, ?> extractKeyFn) { return partitioned(extractKeyFn, defaultPartitioner()); }
/** * Returns an edge with the given source vertex and no destination vertex. * The ordinal of the edge is 0. Typically followed by one of the * {@code to()} method calls. */ @Nonnull public static Edge from(@Nonnull Vertex source) { return from(source, 0); }
.edge(between(tradeSource, slidingStage1) .partitioned(Trade::getTicker, HASH_CODE)) .edge(between(slidingStage1, slidingStage2) .partitioned(entryKey(), HASH_CODE) .distributed()) .edge(between(slidingStage2, formatOutput) .isolated()) .edge(between(formatOutput, sink));
@Override public void addToDag(Planner p) { PlannerVertex pv = p.addVertex(this, p.uniqueVertexName(name()), localParallelism(), processorSupplier); p.addEdges(this, pv.v, e -> e.partitioned(partitionKeyFn).distributed()); } }
private void addSlidingWindowTwoStage(Planner p, SlidingWindowDef wDef) { String vertexName = p.uniqueVertexName(name()); SlidingWindowPolicy winPolicy = wDef.toSlidingWindowPolicy(); Vertex v1 = p.dag.newVertex(vertexName + FIRST_STAGE_VERTEX_NAME_SUFFIX, accumulateByFrameP( nCopies(aggrOp.arity(), constantKey()), nCopies(aggrOp.arity(), (DistributedToLongFunction<JetEvent>) JetEvent::timestamp), TimestampKind.EVENT, winPolicy, aggrOp )); v1.localParallelism(localParallelism()); PlannerVertex pv2 = p.addVertex(this, vertexName, 1, combineToSlidingWindowP(winPolicy, aggrOp, mapToOutputFn.toKeyedWindowResultFn())); p.addEdges(this, v1); p.dag.edge(between(v1, pv2.v).distributed().allToOne()); }
@Override @SuppressWarnings("unchecked") public void addToDag(Planner p) { if (emitsWatermarks || eventTimePolicy == null) { // Reached when the source either emits both JetEvents and watermarks // or neither. In these cases we don't have to insert watermarks. p.addVertex(this, p.uniqueVertexName(name()), localParallelism(), metaSupplierFn.apply(eventTimePolicy != null ? eventTimePolicy : noEventTime()) ); } else { // ------------ // | sourceP | // ------------ // | // isolated // v // ------------- // | insertWmP | // ------------- String v1name = p.uniqueVertexName(name()); Vertex v1 = p.dag.newVertex(v1name, metaSupplierFn.apply(eventTimePolicy)) .localParallelism(localParallelism()); PlannerVertex pv2 = p.addVertex( this, v1name + "-add-timestamps", localParallelism(), insertWatermarksP(eventTimePolicy) ); p.dag.edge(between(v1, pv2.v).isolated()); } }
private void addSlidingWindowSingleStage(Planner p, SlidingWindowDef wDef) { PlannerVertex pv = p.addVertex(this, p.uniqueVertexName(name()), 1, aggregateToSlidingWindowP( nCopies(aggrOp.arity(), constantKey()), nCopies(aggrOp.arity(), (DistributedToLongFunction<JetEvent>) JetEvent::timestamp), TimestampKind.EVENT, wDef.toSlidingWindowPolicy(), aggrOp, mapToOutputFn.toKeyedWindowResultFn() )); p.addEdges(this, pv.v, edge -> edge.distributed().allToOne()); }