public PlannerVertex addVertex(Transform transform, String name, int localParallelism, ProcessorMetaSupplier metaSupplier) { PlannerVertex pv = new PlannerVertex(dag.newVertex(name, metaSupplier)); pv.v.localParallelism(localParallelism); xform2vertex.put(transform, pv); return pv; }
/** * Creates a vertex from a {@code ProcessorSupplier} and adds it to this DAG. * * @see Vertex#Vertex(String, ProcessorSupplier) * * @param name the unique name of the vertex * @param processorSupplier the supplier of {@code Processor} instances which will be used on all members */ @Nonnull public Vertex newVertex(@Nonnull String name, @Nonnull ProcessorSupplier processorSupplier) { return addVertex(new Vertex(name, processorSupplier)); }
private static int determineParallelism(Vertex vertex, int preferredLocalParallelism, int defaultParallelism) { int localParallelism = vertex.getLocalParallelism(); Vertex.checkLocalParallelism(preferredLocalParallelism); Vertex.checkLocalParallelism(localParallelism); return localParallelism != LOCAL_PARALLELISM_USE_DEFAULT ? localParallelism : preferredLocalParallelism != LOCAL_PARALLELISM_USE_DEFAULT ? min(preferredLocalParallelism, defaultParallelism) : defaultParallelism; }
private static int getLocalParallelism(int defaultLocalParallelism, Vertex v) { int localParallelism = v.getLocalParallelism(); if (localParallelism == -1) { localParallelism = v.getMetaSupplier().preferredLocalParallelism(); if (localParallelism == -1) { localParallelism = defaultLocalParallelism; } } return localParallelism; }
protected Edge(@Nonnull Vertex source, int sourceOrdinal, Vertex destination, int destOrdinal) { this.source = source; this.sourceName = source.getName(); this.sourceOrdinal = sourceOrdinal; this.destination = destination; this.destName = destination != null ? destination.getName() : null; this.destOrdinal = destOrdinal; }
for (Entry<String, Integer> entry : vertexIdMap.entrySet()) { final Vertex vertex = dag.getVertex(entry.getKey()); final ProcessorMetaSupplier metaSupplier = vertex.getMetaSupplier(); final int vertexId = entry.getValue(); final int localParallelism = determineParallelism(vertex, metaSupplier.preferredLocalParallelism(), defaultParallelism); final int totalParallelism = localParallelism * clusterSize; final List<EdgeDef> inbound = toEdgeDefs(dag.getInboundEdges(vertex.getName()), defaultEdgeConfig, e -> vertexIdMap.get(e.getSourceName()), isJobDistributed); final List<EdgeDef> outbound = toEdgeDefs(dag.getOutboundEdges(vertex.getName()), defaultEdgeConfig, e -> vertexIdMap.get(e.getDestName()), isJobDistributed); final ILogger logger = nodeEngine.getLogger(String.format("%s.%s#ProcessorMetaSupplier", metaSupplier.getClass().getName(), vertex.getName())); try { metaSupplier.init(new MetaSupplierCtx(instance, jobId, executionId, jobConfig, logger, vertex.getName(), localParallelism, totalParallelism, clusterSize)); } catch (Exception e) { throw sneakyThrow(e); for (Entry<MemberInfo, ExecutionPlan> e : plans.entrySet()) { final ProcessorSupplier processorSupplier = procSupplierFn.apply(e.getKey().getAddress()); checkSerializable(processorSupplier, "ProcessorSupplier in vertex '" + vertex.getName() + '\''); final VertexDef vertexDef = new VertexDef(vertexId, vertex.getName(), processorSupplier, localParallelism); vertexDef.addInboundEdges(inbound); vertexDef.addOutboundEdges(outbound);
@Override public void localParallelism(int localParallelism) { this.localParallelism = Vertex.checkLocalParallelism(localParallelism); }
private void completeVertices(@Nullable Throwable failure) { if (vertices != null) { for (Vertex vertex : vertices) { try { vertex.getMetaSupplier().close(failure); } catch (Exception e) { logger.severe(jobIdString() + " encountered an exception in ProcessorMetaSupplier.complete(), ignoring it", e); } } } }
private boolean containsVertexName(Vertex vertex) { return nameToVertex.containsKey(vertex.getName()); }
/** * Sets the number of processors corresponding to this vertex that will be * created on each member. * <p> * If the value is {@value #LOCAL_PARALLELISM_USE_DEFAULT}, Jet will * determine the vertex's local parallelism during job initialization * from the global default and processor meta-supplier's preferred value. */ @Nonnull public Vertex localParallelism(int localParallelism) { this.localParallelism = checkLocalParallelism(localParallelism); return this; }
@Nonnull private static DAG buildDag(Map<String, Long> counts) { final Pattern delimiter = Pattern.compile("\\W+"); DAG dag = new DAG(); Vertex source = dag.newVertex("source", DocLinesP::new); Vertex tokenize = dag.newVertex("tokenize", flatMapP((String line) -> traverseArray(delimiter.split(line.toLowerCase())) .filter(word -> !word.isEmpty())) ); Vertex aggregate = dag.newVertex("aggregate", aggregateByKeyP(singletonList(wholeItem()), counting(), Util::entry)); Vertex sink = dag.newVertex("sink", () -> new MapSinkP(counts)); return dag.edge(between(source.localParallelism(1), tokenize)) .edge(between(tokenize, aggregate).partitioned(wholeItem(), HASH_CODE)) .edge(between(aggregate, sink)); }
/** * Creates a vertex from a {@code ProcessorMetaSupplier} and adds it to this DAG. * * @see Vertex#Vertex(String, ProcessorMetaSupplier) * * @param name the unique name of the vertex * @param metaSupplier the meta-supplier of {@code ProcessorSupplier}s for each member * */ @Nonnull public Vertex newVertex(@Nonnull String name, @Nonnull ProcessorMetaSupplier metaSupplier) { return addVertex(new Vertex(name, metaSupplier)); }
/** * Sets the local parallelism of the sink. On each member of the cluster * Jet will create this many parallel processors for the sink. To identify * each processor instance, your {@code createFn} can consult {@link * Processor.Context#totalParallelism() procContext.totalParallelism()} and {@link * Processor.Context#globalProcessorIndex() procContext.globalProcessorIndex()}. * Jet calls {@code createFn} exactly once with each {@code * globalProcessorIndex} from 0 to {@code totalParallelism - 1}. * <p> * The default value of this property is 1. */ @Nonnull public SinkBuilder<W, T> preferredLocalParallelism(int preferredLocalParallelism) { Vertex.checkLocalParallelism(preferredLocalParallelism); this.preferredLocalParallelism = preferredLocalParallelism; return this; }
private void addSlidingWindowTwoStage(Planner p, SlidingWindowDef wDef) { String vertexName = p.uniqueVertexName(name()); SlidingWindowPolicy winPolicy = wDef.toSlidingWindowPolicy(); Vertex v1 = p.dag.newVertex(vertexName + FIRST_STAGE_VERTEX_NAME_SUFFIX, accumulateByFrameP( keyFns, nCopies(keyFns.size(), (DistributedToLongFunction<JetEvent>) JetEvent::timestamp), TimestampKind.EVENT, winPolicy, aggrOp)); v1.localParallelism(localParallelism()); PlannerVertex pv2 = p.addVertex(this, vertexName, localParallelism(), combineToSlidingWindowP(winPolicy, aggrOp, mapToOutputFn)); p.addEdges(this, v1, (e, ord) -> e.partitioned(keyFns.get(ord), HASH_CODE)); p.dag.edge(between(v1, pv2.v).distributed().partitioned(entryKey())); }
/** * Sets the destination vertex of this edge, with ordinal 0. */ @Nonnull public Edge to(@Nonnull Vertex destination) { this.destination = destination; this.destName = destination.getName(); return this; }
/** * Creates a vertex from a {@code Supplier<Processor>} and adds it to this DAG. * * @see Vertex#Vertex(String, DistributedSupplier) * * @param name the unique name of the vertex * @param simpleSupplier the simple, parameterless supplier of {@code Processor} instances */ @Nonnull public Vertex newVertex( @Nonnull String name, @Nonnull DistributedSupplier<? extends Processor> simpleSupplier ) { return addVertex(new Vertex(name, simpleSupplier)); }
/** * Factory method that creates a {@link ProcessorMetaSupplier} from the * supplied function that maps a cluster member address to a {@link * ProcessorSupplier}. * * @param addressToSupplier the mapping from address to ProcessorSupplier * @param preferredLocalParallelism the value to return from {@link #preferredLocalParallelism()} */ @Nonnull static ProcessorMetaSupplier of( @Nonnull DistributedFunction<? super Address, ? extends ProcessorSupplier> addressToSupplier, int preferredLocalParallelism ) { Vertex.checkLocalParallelism(preferredLocalParallelism); return new ProcessorMetaSupplier() { @Override public int preferredLocalParallelism() { return preferredLocalParallelism; } @Nonnull @Override public Function<? super Address, ? extends ProcessorSupplier> get(@Nonnull List<Address> addresses) { return addressToSupplier; } }; }
Vertex collector = p.dag.newVertex(collectorName + collectorOrdinal, () -> new HashJoinCollectP(getKeyFn, projectFn)); collector.localParallelism(1); p.dag.edge(from(fromPv.v, fromPv.nextAvailableOrdinal()) .to(collector, 0)
/** * Sets the destination vertex and ordinal of this edge. */ @Nonnull public Edge to(@Nonnull Vertex destination, int ordinal) { this.destination = destination; this.destName = destination.getName(); this.destOrdinal = ordinal; return this; }