/** * Factory method that wraps the given {@code Supplier<Processor>} * and uses it as the supplier of all {@code Processor} instances. * Specifically, returns a meta-supplier that will always return the * result of calling {@link ProcessorSupplier#of(DistributedSupplier)}. * The {@link #preferredLocalParallelism()} of the meta-supplier will be * {@link Vertex#LOCAL_PARALLELISM_USE_DEFAULT}. */ @Nonnull static ProcessorMetaSupplier of(@Nonnull DistributedSupplier<? extends Processor> procSupplier) { return of(procSupplier, Vertex.LOCAL_PARALLELISM_USE_DEFAULT); }
/** * Use {@link SinkProcessors#writeJdbcP}. */ public static <T> ProcessorMetaSupplier metaSupplier( @Nonnull String updateQuery, @Nonnull DistributedSupplier<? extends Connection> connectionSupplier, @Nonnull DistributedBiConsumer<? super PreparedStatement, ? super T> bindFn ) { return ProcessorMetaSupplier.preferLocalParallelismOne(() -> new WriteJdbcP<>(updateQuery, connectionSupplier, bindFn)); }
@Override public void close(Throwable error) throws Exception { wrapped.close(error); } }
/** * Wraps the provided {@code ProcessorMetaSupplier} with a {@code * Supplier<Processor>} that returns processors obtained from it. */ public static Supplier<Processor> supplierFrom(ProcessorMetaSupplier supplier, ProcessorSupplier.Context context) { try { supplier.init(context); } catch (Exception e) { throw sneakyThrow(e); } return supplierFrom(supplier.get(singletonList(LOCAL_ADDRESS)).apply(LOCAL_ADDRESS), context); }
)); return preferredLocalParallelism > 0 ? ProcessorMetaSupplier.of(procSup, preferredLocalParallelism) : ProcessorMetaSupplier.forceTotalParallelismOne(procSup);
public static <T> ProcessorMetaSupplier supplier( @Nonnull String connectionURL, @Nonnull String query, @Nonnull DistributedFunction<? super ResultSet, ? extends T> mapOutputFn ) { return ProcessorMetaSupplier.forceTotalParallelismOne(ProcessorSupplier.of(() -> new ReadJdbcP<>( () -> DriverManager.getConnection(connectionURL), (connection, parallelism, index) -> { PreparedStatement statement = connection.prepareStatement(query); try { return statement.executeQuery(); } catch (SQLException e) { statement.close(); throw e; } }, mapOutputFn) )); }
final int vertexId = entry.getValue(); final int localParallelism = determineParallelism(vertex, metaSupplier.preferredLocalParallelism(), defaultParallelism); final int totalParallelism = localParallelism * clusterSize; final List<EdgeDef> inbound = toEdgeDefs(dag.getInboundEdges(vertex.getName()), defaultEdgeConfig, metaSupplier.getClass().getName(), vertex.getName())); try { metaSupplier.init(new MetaSupplierCtx(instance, jobId, executionId, jobConfig, logger, vertex.getName(), localParallelism, totalParallelism, clusterSize)); } catch (Exception e) { Function<? super Address, ? extends ProcessorSupplier> procSupplierFn = metaSupplier.get(addresses); for (Entry<MemberInfo, ExecutionPlan> e : plans.entrySet()) { final ProcessorSupplier processorSupplier = procSupplierFn.apply(e.getKey().getAddress());
/** * Variant of {@link #forceTotalParallelismOne(ProcessorSupplier, String)} where the node * for the supplier will be chosen randomly. */ @Nonnull static ProcessorMetaSupplier forceTotalParallelismOne(@Nonnull ProcessorSupplier supplier) { return forceTotalParallelismOne(supplier, newUnsecureUuidString()); }
@Override public int preferredLocalParallelism() { return wrapped.preferredLocalParallelism(); }
@Override public void init(@Nonnull Context context) throws Exception { wrapped.init(context); }
@Nonnull @Override public Function<Address, ProcessorSupplier> get(@Nonnull List<Address> addresses) { Function<? super Address, ? extends ProcessorSupplier> function = wrapped.get(addresses); return address -> new WrappingProcessorSupplier(function.apply(address), wrapperSupplier); }
null)); return preferredLocalParallelism != 0 ? ProcessorMetaSupplier.of(procSup, preferredLocalParallelism) : ProcessorMetaSupplier.forceTotalParallelismOne(procSup);
/** * Returns a supplier of processors for {@link Sources#jmsTopicBuilder}. */ @Nonnull public static <T> ProcessorMetaSupplier streamJmsTopicP( @Nonnull DistributedSupplier<? extends Connection> connectionSupplier, @Nonnull DistributedFunction<? super Connection, ? extends Session> sessionFn, @Nonnull DistributedFunction<? super Session, ? extends MessageConsumer> consumerFn, @Nonnull DistributedConsumer<? super Session> flushFn, @Nonnull DistributedFunction<? super Message, ? extends T> projectionFn, @Nonnull EventTimePolicy<? super T> eventTimePolicy) { return ProcessorMetaSupplier.forceTotalParallelismOne( StreamJmsP.supplier(connectionSupplier, sessionFn, consumerFn, flushFn, projectionFn, eventTimePolicy)); }
private static int getLocalParallelism(int defaultLocalParallelism, Vertex v) { int localParallelism = v.getLocalParallelism(); if (localParallelism == -1) { localParallelism = v.getMetaSupplier().preferredLocalParallelism(); if (localParallelism == -1) { localParallelism = defaultLocalParallelism; } } return localParallelism; }
/** * Wraps the provided {@code ProcessorSupplier} into a meta-supplier that * will always return it. The {@link #preferredLocalParallelism()} of * the meta-supplier will be one, i.e., no local parallelization. * <p> * The parallelism will be overriden if the {@link Vertex#localParallelism(int)} is * set to a specific value. */ @Nonnull static ProcessorMetaSupplier preferLocalParallelismOne(@Nonnull ProcessorSupplier supplier) { return of(supplier, 1); }
/** * Internal API, use {@link SourceProcessors#streamSocketP(String, int, Charset)}. */ public static ProcessorMetaSupplier supplier(String host, int port, @Nonnull String charset) { return ProcessorMetaSupplier.preferLocalParallelismOne( () -> new StreamSocketP(host, port, Charset.forName(charset))); } }
public static ProcessorMetaSupplier metaSupplier(String listName, ClientConfig clientConfig) { String clientXml = asXmlString(clientConfig); return forceTotalParallelismOne( ProcessorSupplier.of(() -> { return new ReadIListP(listName, clientXml); }), listName ); } }
private void completeVertices(@Nullable Throwable failure) { if (vertices != null) { for (Vertex vertex : vertices) { try { vertex.getMetaSupplier().close(failure); } catch (Exception e) { logger.severe(jobIdString() + " encountered an exception in ProcessorMetaSupplier.complete(), ignoring it", e); } } } }
/** * Creates a vertex from a {@code ProcessorSupplier}. * * @param name the unique name of the vertex. This name identifies the vertex in the snapshot * @param processorSupplier the supplier of {@code Processor} instances which will be used on all members */ public Vertex(@Nonnull String name, @Nonnull ProcessorSupplier processorSupplier) { this(name, ProcessorMetaSupplier.of(processorSupplier)); }