@Nonnull private static DAG buildDag(Map<String, Long> counts) { final Pattern delimiter = Pattern.compile("\\W+"); DAG dag = new DAG(); Vertex source = dag.newVertex("source", DocLinesP::new); Vertex tokenize = dag.newVertex("tokenize", flatMapP((String line) -> traverseArray(delimiter.split(line.toLowerCase())) .filter(word -> !word.isEmpty())) ); Vertex aggregate = dag.newVertex("aggregate", aggregateByKeyP(singletonList(wholeItem()), counting(), Util::entry)); Vertex sink = dag.newVertex("sink", () -> new MapSinkP(counts)); return dag.edge(between(source.localParallelism(1), tokenize)) .edge(between(tokenize, aggregate).partitioned(wholeItem(), HASH_CODE)) .edge(between(aggregate, sink)); }
/** * Adds a vertex to this DAG. The vertex name must be unique. */ @Nonnull public DAG vertex(@Nonnull Vertex vertex) { addVertex(vertex); return this; }
private void rewriteDagWithSnapshotRestore(DAG dag, long snapshotId, String mapName) { IMap<Object, Object> snapshotMap = nodeEngine.getHazelcastInstance().getMap(mapName); snapshotId = SnapshotValidator.validateSnapshot(snapshotId, jobIdString(), snapshotMap); logger.info("State of " + jobIdString() + " will be restored from snapshot " + snapshotId + ", map=" + mapName); List<Vertex> originalVertices = new ArrayList<>(); dag.iterator().forEachRemaining(originalVertices::add); Map<String, Integer> vertexToOrdinal = new HashMap<>(); Vertex readSnapshotVertex = dag.newVertex(SNAPSHOT_VERTEX_PREFIX + "read", readMapP(mapName)); long finalSnapshotId = snapshotId; Vertex explodeVertex = dag.newVertex(SNAPSHOT_VERTEX_PREFIX + "explode", () -> new ExplodeSnapshotP(vertexToOrdinal, finalSnapshotId)); dag.edge(between(readSnapshotVertex, explodeVertex).isolated()); int index = 0; // add the edges for (Vertex userVertex : originalVertices) { vertexToOrdinal.put(userVertex.getName(), index); int destOrdinal = dag.getInboundEdges(userVertex.getName()).size(); dag.edge(new SnapshotRestoreEdge(explodeVertex, index, userVertex, destOrdinal)); index++; } }
private void addSlidingWindowTwoStage(Planner p, SlidingWindowDef wDef) { String vertexName = p.uniqueVertexName(name()); SlidingWindowPolicy winPolicy = wDef.toSlidingWindowPolicy(); Vertex v1 = p.dag.newVertex(vertexName + FIRST_STAGE_VERTEX_NAME_SUFFIX, accumulateByFrameP( keyFns, nCopies(keyFns.size(), (DistributedToLongFunction<JetEvent>) JetEvent::timestamp), TimestampKind.EVENT, winPolicy, aggrOp)); v1.localParallelism(localParallelism()); PlannerVertex pv2 = p.addVertex(this, vertexName, localParallelism(), combineToSlidingWindowP(winPolicy, aggrOp, mapToOutputFn)); p.addEdges(this, v1, (e, ord) -> e.partitioned(keyFns.get(ord), HASH_CODE)); p.dag.edge(between(v1, pv2.v).distributed().partitioned(entryKey())); }
public PlannerVertex addVertex(Transform transform, String name, int localParallelism, ProcessorMetaSupplier metaSupplier) { PlannerVertex pv = new PlannerVertex(dag.newVertex(name, metaSupplier)); pv.v.localParallelism(localParallelism); xform2vertex.put(transform, pv); return pv; }
public void addEdges(Transform transform, Vertex toVertex, BiConsumer<Edge, Integer> configureEdgeFn) { int destOrdinal = 0; for (Transform fromTransform : transform.upstream()) { PlannerVertex fromPv = xform2vertex.get(fromTransform); Edge edge = from(fromPv.v, fromPv.nextAvailableOrdinal()).to(toVertex, destOrdinal); dag.edge(edge); configureEdgeFn.accept(edge, destOrdinal); destOrdinal++; } }
@Override public IdentifiedDataSerializable create(int typeId) { switch (typeId) { case DAG: return new DAG(); case EDGE: return new Edge(); case VERTEX: return new Vertex(); case APPLY_FN_ENTRY_PROCESSOR: return new ApplyFnEntryProcessor(); default: throw new IllegalArgumentException("Unknown type id " + typeId); } } }
+ "' and '" + edge.getDestName() + '\''); if (!containsVertex(edge.getSource())) { throw new IllegalArgumentException( containsVertexName(edge.getSource()) ? "This DAG has a vertex called '" + edge.getSourceName() + "', but the supplied edge's source is a different vertex with the same name" ); if (!containsVertex(edge.getDestination())) { throw new IllegalArgumentException( containsVertexName(edge.getDestination()) ? "This DAG has a vertex called '" + edge.getDestName() + "', but the supplied edge's destination is a different vertex with the same name" : "Destination vertex '" + edge.getDestName() + "' is not in this DAG"); if (getInboundEdges(edge.getDestName()) .stream().anyMatch(e -> e.getDestOrdinal() == edge.getDestOrdinal())) { throw new IllegalArgumentException("Vertex '" + edge.getDestName() ? ", use Edge.from().to() to specify another ordinal" : "")); if (getOutboundEdges(edge.getSourceName()) .stream().anyMatch(e -> e.getSourceOrdinal() == edge.getSourceOrdinal())) { throw new IllegalArgumentException("Vertex '" + edge.getSourceName()
private void addToDagTwoStage(Planner p) { String vertexName = p.uniqueVertexName(name()); Vertex v1 = p.dag.newVertex(vertexName + FIRST_STAGE_VERTEX_NAME_SUFFIX, accumulateP(aggrOp)) .localParallelism(localParallelism()); PlannerVertex pv2 = p.addVertex(this, vertexName, 1, combineP(aggrOp)); p.addEdges(this, v1); p.dag.edge(between(v1, pv2.v).distributed().allToOne()); } }
public static CompletableFuture<Void> copyMapUsingJob(JetInstance instance, int queueSize, String sourceMap, String targetMap) { DAG dag = new DAG(); Vertex source = dag.newVertex("readMap(" + sourceMap + ')', readMapP(sourceMap)); Vertex sink = dag.newVertex("writeMap(" + targetMap + ')', writeMapP(targetMap)); dag.edge(between(source, sink).setConfig(new EdgeConfig().setQueueSize(queueSize))); JobConfig jobConfig = new JobConfig() .setName("copy-" + sourceMap + "-to-" + targetMap); return instance.newJob(dag, jobConfig).getFuture(); } }
Vertex joiner = p.addVertex(this, namePrefix + "-joiner", localParallelism(), () -> new HashJoinP<>(keyFns, tags, mapToOutputBiFn, mapToOutputTriFn)).v; p.dag.edge(from(primary.v, primary.nextAvailableOrdinal()).to(joiner, 0)); DistributedFunction<Object, Object> projectFn = (DistributedFunction<Object, Object>) clause.rightProjectFn(); Vertex collector = p.dag.newVertex(collectorName + collectorOrdinal, () -> new HashJoinCollectP(getKeyFn, projectFn)); collector.localParallelism(1); p.dag.edge(from(fromPv.v, fromPv.nextAvailableOrdinal()) .to(collector, 0) .distributed().broadcast()); p.dag.edge(from(collector, 0) .to(joiner, collectorOrdinal) .broadcast().priority(-1));
/** * Creates a vertex from a {@code ProcessorSupplier} and adds it to this DAG. * * @see Vertex#Vertex(String, ProcessorSupplier) * * @param name the unique name of the vertex * @param processorSupplier the supplier of {@code Processor} instances which will be used on all members */ @Nonnull public Vertex newVertex(@Nonnull String name, @Nonnull ProcessorSupplier processorSupplier) { return addVertex(new Vertex(name, processorSupplier)); }
public static void main(String[] args) throws Exception { System.setProperty("hazelcast.logging.type", "log4j"); Jet.newJetInstance(); JetInstance jet = Jet.newJetInstance(); try { IMapJet<Object, Object> map = jet.getMap("map"); range(0, COUNT).parallel().forEach(i -> map.put("key-" + i, i)); DAG dag = new DAG(); Vertex source = dag.newVertex("map-source", SourceProcessors.readMapP(map.getName())); Vertex sink = dag.newVertex("file-sink", new WriteFilePSupplier(OUTPUT_FOLDER)); dag.edge(between(source, sink)); jet.newJob(dag).join(); System.out.println("\nHazelcast IMap dumped to folder " + new File(OUTPUT_FOLDER).getAbsolutePath()); } finally { Jet.shutdownAll(); } } }
private void addToDagTwoStage(Planner p) { List<DistributedFunction<?, ? extends K>> groupKeyFns = this.groupKeyFns; String vertexName = p.uniqueVertexName(this.name()); Vertex v1 = p.dag.newVertex(vertexName + FIRST_STAGE_VERTEX_NAME_SUFFIX, accumulateByKeyP(groupKeyFns, aggrOp)) .localParallelism(localParallelism()); PlannerVertex pv2 = p.addVertex(this, vertexName, localParallelism(), combineByKeyP(aggrOp, mapToOutputFn)); p.addEdges(this, v1, (e, ord) -> e.partitioned(groupKeyFns.get(ord), HASH_CODE)); p.dag.edge(between(v1, pv2.v).distributed().partitioned(entryKey())); } }
/** * Creates a vertex from a {@code ProcessorMetaSupplier} and adds it to this DAG. * * @see Vertex#Vertex(String, ProcessorMetaSupplier) * * @param name the unique name of the vertex * @param metaSupplier the meta-supplier of {@code ProcessorSupplier}s for each member * */ @Nonnull public Vertex newVertex(@Nonnull String name, @Nonnull ProcessorMetaSupplier metaSupplier) { return addVertex(new Vertex(name, metaSupplier)); }
private static DAG buildDag() { DAG dag = new DAG(); SlidingWindowPolicy winPolicy = slidingWinPolicy(SLIDING_WINDOW_LENGTH_MILLIS, SLIDE_STEP_MILLIS); Vertex streamTrades = dag.newVertex("stream-trades", SourceProcessors.<Trade, Long, Trade>streamMapP(TRADES_MAP_NAME, DistributedPredicate.alwaysTrue(), EventJournalMapEvent::getNewValue, JournalInitialPosition.START_FROM_OLDEST, 30000L ))); Vertex slidingWindow = dag.newVertex("aggregate-to-sliding-win", aggregateToSlidingWindowP( singletonList((DistributedFunction<Trade, String>) Trade::getTicker), counting(), TimestampedEntry::fromWindowResult)); Vertex formatOutput = dag.newVertex("format-output", formatOutput()); Vertex sink = dag.newVertex("sink", writeFileP(OUTPUT_DIR_NAME, Object::toString, StandardCharsets.UTF_8, false)); .edge(between(streamTrades, slidingWindow) .partitioned(Trade::getTicker, HASH_CODE) .distributed()) .edge(between(slidingWindow, formatOutput).isolated()) .edge(between(formatOutput, sink));
@Override public void addToDag(Planner p) { String vertexName = p.uniqueVertexName(this.name()); Vertex v1 = p.dag.newVertex(vertexName + FIRST_STAGE_VERTEX_NAME_SUFFIX, distinctP(keyFn)) .localParallelism(localParallelism()); PlannerVertex pv2 = p.addVertex(this, vertexName, localParallelism(), distinctP(keyFn)); p.addEdges(this, v1, (e, ord) -> e.partitioned(keyFn, HASH_CODE)); p.dag.edge(between(v1, pv2.v).distributed().partitioned(keyFn)); }
/** * Creates a vertex from a {@code Supplier<Processor>} and adds it to this DAG. * * @see Vertex#Vertex(String, DistributedSupplier) * * @param name the unique name of the vertex * @param simpleSupplier the simple, parameterless supplier of {@code Processor} instances */ @Nonnull public Vertex newVertex( @Nonnull String name, @Nonnull DistributedSupplier<? extends Processor> simpleSupplier ) { return addVertex(new Vertex(name, simpleSupplier)); }
final Pattern delimiter = Pattern.compile("\\W+"); DAG dag = new DAG(); Vertex source = dag.newVertex("source", readMapP(DOCID_NAME)); Vertex docLines = dag.newVertex("doc-lines", Vertex tokenize = dag.newVertex("tokenize", flatMapP((String line) -> traverseArray(delimiter.split(line.toLowerCase())) .filter(word -> !word.isEmpty())) ); Vertex accumulate = dag.newVertex("accumulate", accumulateByKeyP(singletonList(wholeItem()), counting())); Vertex combine = dag.newVertex("combine", combineByKeyP(counting(), Util::entry)); Vertex sink = dag.newVertex("sink", writeMapP("counts")); return dag.edge(between(source.localParallelism(1), docLines)) .edge(between(docLines.localParallelism(1), tokenize)) .edge(between(tokenize, accumulate) .partitioned(wholeItem(), HASH_CODE)) .edge(between(accumulate, combine) .distributed() .partitioned(entryKey())) .edge(between(combine, sink));
private void addSlidingWindowTwoStage(Planner p, SlidingWindowDef wDef) { String vertexName = p.uniqueVertexName(name()); SlidingWindowPolicy winPolicy = wDef.toSlidingWindowPolicy(); Vertex v1 = p.dag.newVertex(vertexName + FIRST_STAGE_VERTEX_NAME_SUFFIX, accumulateByFrameP( nCopies(aggrOp.arity(), constantKey()), nCopies(aggrOp.arity(), (DistributedToLongFunction<JetEvent>) JetEvent::timestamp), TimestampKind.EVENT, winPolicy, aggrOp )); v1.localParallelism(localParallelism()); PlannerVertex pv2 = p.addVertex(this, vertexName, 1, combineToSlidingWindowP(winPolicy, aggrOp, mapToOutputFn.toKeyedWindowResultFn())); p.addEdges(this, v1); p.dag.edge(between(v1, pv2.v).distributed().allToOne()); }