@Nonnull private static ContextFactory<SentimentAnalyzer> sentimentAnalyzerContext() { return ContextFactory .withCreateFn(jet -> new SentimentAnalyzer()) .shareLocally(); }
/** * Returns a copy of this {@link ContextFactory} with the * <em>shareLocally</em> flag set. If the pipeline doesn't have grouping, * there will be: * <ul> * <li>one context object per local processor, if flag is disabled * <li>one context object per member, if flag is enabled. Make * sure the context object is <em>thread-safe</em> in this case. * </ul> * * @return a copy of this factory with the {@code isSharedLocally} flag set. */ @Nonnull public ContextFactory<C> shareLocally() { return new ContextFactory<>(createFn, destroyFn, isCooperative, true, maxPendingCallsPerProcessor, orderedAsyncResponses); }
@Override protected void init(@Nonnull Context context) { if (!contextFactory.isSharedLocally()) { assert contextObject == null : "contextObject is not null: " + contextObject; contextObject = contextFactory.createFn().apply(context.jetInstance()); } maxAsyncOps = contextFactory.getMaxPendingCallsPerProcessor(); queue = new ArrayDeque<>(maxAsyncOps); }
@Override public void close() { // close() might be called even if init() was not called. // Only destroy the context if is not shared (i.e. it is our own). if (contextObject != null && !contextFactory.isSharedLocally()) { contextFactory.destroyFn().accept(contextObject); } contextObject = null; }
@SuppressWarnings("unchecked") private static <T, K> ProcessorSupplier distinctP(DistributedFunction<? super T, ? extends K> keyFn) { return filterUsingContextP(ContextFactory.withCreateFn(jet -> new HashSet<>()), (seenItems, item) -> seenItems.add(keyFn.apply((T) item))); } }
ContextFactory.withCreateFn(jet -> null).nonCooperative(), (Object ctx, Entry<?, String> e) -> traverseStream(docLines(e.getValue()))) );
@Override public void init(@Nonnull Context context) { if (contextFactory.isSharedLocally()) { contextObject = contextFactory.createFn().apply(context.jetInstance()); } }
@Override public void close(Throwable error) { if (contextObject != null) { contextFactory.destroyFn().accept(contextObject); } }
/** * Loads the pre-trained model from the specified path * * @param modelPath path of the model */ private static ContextFactory<ImageClassifierVggCifar10> classifierContext(String modelPath) { return ContextFactory.withCreateFn(jet -> { ImageClassifierVggCifar10 classifier = new ImageClassifierVggCifar10(); classifier.loadModel(new File(modelPath)); return classifier; }); } }
ContextFactory.withCreateFn(jet -> null).nonCooperative(), (Object ctx, Entry<Long, String> e) -> traverseStream(docLines("books/" + e.getValue())
@Override public void close() { // close() might be called even if init() was not called. // Only destroy the context if is not shared (i.e. it is our own). if (contextObject != null && !contextFactory.isSharedLocally()) { contextFactory.destroyFn().accept(contextObject); } contextObject = null; }
@Override protected void init(@Nonnull Context context) { if (!contextFactory.isSharedLocally()) { assert contextObject == null : "contextObject is not null: " + contextObject; contextObject = contextFactory.createFn().apply(context.jetInstance()); } }
/** * Returns a factory that provides a {@link ReplicatedMap} as the context * object. A replicated map is a particularly good choice if you are * enriching an event stream with the data stored in the Hazelcast Jet * cluster. Unlike in a {@code hashJoin} transformation, the data in the * map can change while the job is running so you can keep the enriching * dataset up-to-date. Unlike {@code IMap}, the data you access is local so * you won't do any blocking calls using it (important for performance). * <p> * If you want to destroy the map after the job finishes, call * {@code factory.destroyFn(ReplicatedMap::destroy)} on the object you get * from this method. * <p> * Example usage (without destroyFn): * <pre> * p.drawFrom( /* a batch or streaming source */ ) * .mapUsingContext(replicatedMapContext("fooMapName"), * (map, item) -> tuple2(item, map.get(item.getKey()))) * .destroyFn(ReplicatedMap::destroy); * </pre> * * @param mapName name of the {@link ReplicatedMap} to use as the context * @param <K> type of the map key * @param <V> type of the map value */ @Nonnull public static <K, V> ContextFactory<ReplicatedMap<K, V>> replicatedMapContext(@Nonnull String mapName) { return ContextFactory .withCreateFn(jet -> jet.getHazelcastInstance().getReplicatedMap(mapName)); }
/** * Returns a factory that provides an {@link IMapJet} as the context. This * is useful if you are enriching an event stream with the data stored in * the Hazelcast Jet cluster. Unlike in a {@code hashJoin} transformation, * the data in the map can change while the job is running so you can keep * the enriching dataset up-to-date. * <p> * Instead of using this factory, you can call {@link * GeneralStage#mapUsingIMapAsync(IMap, DistributedBiFunction)} or {@link * GeneralStageWithKey#mapUsingIMapAsync(IMap, DistributedBiFunction)}. * <p> * If you plan to use a sync method on the map, call {@link * ContextFactory#nonCooperative()} on the returned factory. * * @param mapName name of the map used as context * @param <K> key type * @param <V> value type * @return the context factory */ @Nonnull public static <K, V> ContextFactory<IMapJet<K, V>> iMapContext(@Nonnull String mapName) { return ContextFactory .withCreateFn(jet -> jet.<K, V>getMap(mapName)) .shareLocally(); }
@Override protected void init(@Nonnull Context context) { if (!contextFactory.isSharedLocally()) { assert contextObject == null : "contextObject is not null: " + contextObject; contextObject = contextFactory.createFn().apply(context.jetInstance()); } maxAsyncOps = contextFactory.getMaxPendingCallsPerProcessor(); resultQueue = new ManyToOneConcurrentArrayQueue<>(maxAsyncOps); }
@Override public void close() { // close() might be called even if init() was not called. // Only destroy the context if is not shared (i.e. it is our own). if (contextObject != null && !contextFactory.isSharedLocally()) { contextFactory.destroyFn().accept(contextObject); } contextObject = null; }
/** * Returns a copy of this {@link ContextFactory} with the * <em>isCooperative</em> flag set to {@code false}. The context factory is * cooperative by default. Call this method if your transform function * doesn't follow the {@linkplain Processor#isCooperative() cooperative * processor contract}, that is if it waits for IO, blocks for * synchronization, takes too long to complete etc. If you intend to use * the factory for an async operation, you also typically can use a * cooperative processor. Cooperative processors offer higher performance. * * @return a copy of this factory with the {@code isCooperative} flag set * to {@code false}. */ @Nonnull public ContextFactory<C> nonCooperative() { return new ContextFactory<>(createFn, destroyFn, false, isSharedLocally, maxPendingCallsPerProcessor, orderedAsyncResponses); }
Processors.combineToSlidingWindowP(winPolicy, counting(), TimestampedEntry::fromWindowResult)); Vertex formatOutput = dag.newVertex("format-output", mapUsingContextP( ContextFactory.withCreateFn(x -> DateTimeFormatter.ofPattern("HH:mm:ss.SSS")), (DateTimeFormatter timeFormat, TimestampedEntry<String, Long> tse) -> String.format("%s %5s %4d",
return new ContextFactory<>(createFn, destroyFn, isCooperative, isSharedLocally, maxPendingCallsPerProcessor, false);
/** * Returns a copy of this {@link ContextFactory} with the destroy-function * replaced with the given function. * <p> * The destroy function is called at the end of the job to destroy all * created context objects. * * @param destroyFn the function to destroy user-defined context * @return a copy of this factory with the supplied destroy-function */ @Nonnull public ContextFactory<C> withDestroyFn(@Nonnull DistributedConsumer<? super C> destroyFn) { checkSerializable(destroyFn, "destroyFn"); return new ContextFactory<>(createFn, destroyFn, isCooperative, isSharedLocally, maxPendingCallsPerProcessor, orderedAsyncResponses); }