/** * Variant of {@link #preferLocalParallelismOne(ProcessorSupplier)} where * the supplied {@code DistributedSupplier<Processor>} will be * wrapped into a {@link ProcessorSupplier}. */ @Nonnull static ProcessorMetaSupplier preferLocalParallelismOne( @Nonnull DistributedSupplier<? extends Processor> procSupplier ) { return of(ProcessorSupplier.of(procSupplier), 1); }
@Override public void close(Throwable error) throws Exception { wrapped.close(error); } }
/** * Wraps the provided {@code ProcessorSupplier} with a {@code * Supplier<Processor>} that returns processors obtained from it. */ public static Supplier<Processor> supplierFrom(ProcessorSupplier supplier, ProcessorSupplier.Context context) { try { supplier.init(context); } catch (Exception e) { throw sneakyThrow(e); } return () -> supplier.get(1).iterator().next(); }
private Processor newProcessorFromSupplier() { return supplier.get(1).iterator().next(); }
@Override public void init(@Nonnull Context context) throws Exception { wrapped.init(context); }
@Nonnull @Override public Collection<? extends Processor> get(int count) { Collection<? extends Processor> processors = wrapped.get(count); return processors.stream() .map(wrapperSupplier) .collect(toList()); }
private void initProcSuppliers(long jobId, long executionId) { JetService service = nodeEngine.getService(JetService.SERVICE_NAME); for (VertexDef vertex : vertices) { ProcessorSupplier supplier = vertex.processorSupplier(); ILogger logger = nodeEngine.getLogger(supplier.getClass().getName() + '.' + vertex.name() + "#ProcessorSupplier"); try { supplier.init(new ProcSupplierCtx( service.getJetInstance(), jobId, executionId, jobConfig, logger, vertex.name(), vertex.localParallelism(), vertex.localParallelism() * memberCount, memberIndex, memberCount )); } catch (Exception e) { throw sneakyThrow(e); } } }
public static ProcessorTransform customProcessorTransform( @Nonnull String name, @Nonnull Transform upstream, @Nonnull DistributedSupplier<Processor> createProcessorFn ) { return new ProcessorTransform(name, upstream, ProcessorSupplier.of(createProcessorFn)); }
private static Collection<? extends Processor> createProcessors(VertexDef vertexDef, int parallelism) { final Collection<? extends Processor> processors = vertexDef.processorSupplier().get(parallelism); if (processors.size() != parallelism) { throw new JetException("ProcessorSupplier failed to return the requested number of processors." + " Requested: " + parallelism + ", returned: " + processors.size()); } return processors; }
/** * Complete local execution. If local execution was started, it should be * called after execution has completed. */ public void completeExecution(Throwable error) { assert executionFuture == null || executionFuture.isDone() : "If execution was begun, then completeExecution() should not be called before execution is done."; for (Tasklet tasklet : tasklets) { try { tasklet.close(); } catch (Throwable e) { logger.severe(jobNameAndExecutionId() + " encountered an exception in Processor.close(), ignoring it", e); } } for (ProcessorSupplier s : procSuppliers) { try { s.close(error); } catch (Throwable e) { logger.severe(jobNameAndExecutionId() + " encountered an exception in ProcessorSupplier.complete(), ignoring it", e); } } MetricsRegistry metricsRegistry = ((NodeEngineImpl) nodeEngine).getMetricsRegistry(); processors.forEach(metricsRegistry::deregister); tasklets.forEach(metricsRegistry::deregister); }
/** * Factory method that wraps the given {@code Supplier<Processor>} * and uses it as the supplier of all {@code Processor} instances. * Specifically, returns a meta-supplier that will always return the * result of calling {@link ProcessorSupplier#of(DistributedSupplier)}. * * @param procSupplier the supplier of processors * @param preferredLocalParallelism the value to return from {@link #preferredLocalParallelism()} */ @Nonnull static ProcessorMetaSupplier of( @Nonnull DistributedSupplier<? extends Processor> procSupplier, int preferredLocalParallelism ) { return of(ProcessorSupplier.of(procSupplier), preferredLocalParallelism); }
public static <T> ProcessorMetaSupplier supplier( @Nonnull String connectionURL, @Nonnull String query, @Nonnull DistributedFunction<? super ResultSet, ? extends T> mapOutputFn ) { return ProcessorMetaSupplier.forceTotalParallelismOne(ProcessorSupplier.of(() -> new ReadJdbcP<>( () -> DriverManager.getConnection(connectionURL), (connection, parallelism, index) -> { PreparedStatement statement = connection.prepareStatement(query); try { return statement.executeQuery(); } catch (SQLException e) { statement.close(); throw e; } }, mapOutputFn) )); }
public static ProcessorMetaSupplier streamTwitterP(Properties properties, List<String> terms) { return preferLocalParallelismOne(ProcessorSupplier.of(() -> new StreamTwitterP(properties, terms))); }
public static ProcessorMetaSupplier metaSupplier(String listName, ClientConfig clientConfig) { String clientXml = asXmlString(clientConfig); return forceTotalParallelismOne( ProcessorSupplier.of(() -> { return new ReadIListP(listName, clientXml); }), listName ); } }
checkSerializable(destroyFn, "destroyFn"); checkNotNegative(preferredLocalParallelism + 1, "preferredLocalParallelism must >= -1"); ProcessorSupplier procSup = ProcessorSupplier.of( () -> new ConvenientSourceP<>( createFn,
checkSerializable(destroyFn, "destroyFn"); checkNotNegative(preferredLocalParallelism + 1, "preferredLocalParallelism must >= -1"); ProcessorSupplier procSup = ProcessorSupplier.of( () -> new ConvenientSourceP<S, T>( createFn,