@RequestMapping("/wordCount") public CommandResponse wordCount(@RequestParam(value = "sourceName") String sourceName, @RequestParam(value = "sinkName") String sinkName) { JobConfig jobConfig = new JobConfig(); jetClient.newJob(PipelineBuilder.buildPipeline(sourceName, sinkName), jobConfig).join(); IMap<String, Long> counts = jetClient.getMap(sinkName); List<Map.Entry<String, Long>> topResult = counts.entrySet() .stream() .sorted(comparingLong(Map.Entry<String, Long>::getValue).reversed()) .limit(1) .collect(Collectors.toList()); Map.Entry<String, Long> entry = topResult.get(0); return new CommandResponse("Top word is `" + entry.getKey() + "` with the count: " + entry.getValue()); }
@Override public HazelcastInstance getClient() { return jet.getHazelcastInstance(); }
private Thread shutdownHookThread(NodeEngine nodeEngine) { return new Thread(() -> { String policy = nodeEngine.getProperties().getString(SHUTDOWNHOOK_POLICY); if (policy.equals("TERMINATE")) { jetInstance.getHazelcastInstance().getLifecycleService().terminate(); } else { jetInstance.shutdown(); } }, "jet.ShutdownThread"); } }
/** * Creates and returns a Jet job based on the supplied DAG. Jet will * asynchronously start executing the job. */ @Nonnull default Job newJob(@Nonnull DAG dag) { return newJob(dag, new JobConfig()); }
public static void main(String[] args) { ApplicationContext context = new GenericXmlApplicationContext("application-context-with-schema.xml"); JetInstance jetInstance = (JetInstance) context.getBean("instance"); JetInstance jetClient = (JetInstance) context.getBean("client"); Pipeline pipeline = Pipeline.create(); pipeline.drawFrom(CustomSourceP.customSource()) .drainTo(Sinks.logger()); JobConfig jobConfig = new JobConfig() .addClass(XmlConfigurationWithSchemaSample.class) .addClass(CustomSourceP.class); jetClient.newJob(pipeline, jobConfig).join(); jetClient.shutdown(); jetInstance.shutdown(); } }
public static void main(String[] args) throws Exception { System.setProperty("hazelcast.logging.type", "log4j"); JetConfig jetConfig = getJetConfig(); JetInstance jet = Jet.newJetInstance(jetConfig); Jet.newJetInstance(jetConfig); try { Pipeline p = Pipeline.create(); p.drawFrom(Sources.<Integer, Integer>mapJournal(MAP_NAME, START_FROM_OLDEST)) .map(Entry::getValue) .drainTo(Sinks.list(SINK_NAME)); jet.newJob(p); IMapJet<Integer, Integer> map = jet.getMap(MAP_NAME); for (int i = 0; i < 1000; i++) { map.set(i, i); } TimeUnit.SECONDS.sleep(3); System.out.println("Read " + jet.getList(SINK_NAME).size() + " entries from map journal."); } finally { Jet.shutdownAll(); } }
public static void main(String[] args) throws Exception { System.setProperty("hazelcast.logging.type", "log4j"); JetInstance jet = Jet.newJetInstance(); try { IList<Integer> inputList = jet.getList(INPUT_LIST); for (int i = 0; i < ITEM_COUNT; i++) { inputList.add(i); } Pipeline p = Pipeline.create(); p.drawFrom(Sources.<Integer>list(INPUT_LIST)) .map(i -> "item" + i) .drainTo(Sinks.list(RESULT_LIST)); jet.newJob(p).join(); IList<String> outputList = jet.getList(RESULT_LIST); System.out.println("Result list items: " + new ArrayList<>(outputList)); } finally { Jet.shutdownAll(); } } }
public static void main(String[] args) { System.setProperty("hazelcast.logging.type", "log4j"); JetInstance jet = JetBootstrap.getInstance(); jet.getHazelcastInstance().getConfig().addEventJournalConfig(new EventJournalConfig() .setMapName(TRADES_MAP_NAME) .setCapacity(TRADES_PER_SEC * 10)); startConsolePrinterThread(jet, VOLUME_MAP_NAME); try { JobConfig jobConfig = new JobConfig(); jobConfig.setName("Trade Volume"); jet.newJob(buildPipeline(), jobConfig); TradeGenerator.generate(NUMBER_OF_TICKERS, jet.getMap(TRADES_MAP_NAME), TRADES_PER_SEC); } finally { stopConsolePrinterThread(); Jet.shutdownAll(); } } }
/** * Creates a Hazelcast Jet cluster, attaches a topic listener and runs the pipeline */ public static void main(String[] args) { System.setProperty("hazelcast.logging.type", "log4j"); try { System.out.println("Creating Jet instance 1"); JetInstance jet = Jet.newJetInstance(); System.out.println("Creating Jet instance 2"); Jet.newJetInstance(); System.out.println("Configure Topic Listener"); ITopic<String> topic = jet.getHazelcastInstance().getTopic(TOPIC_NAME); addListener(topic, e -> System.out.println("Line starts with `The`: " + e)); System.out.println("\nRunning the pipeline"); Pipeline p = buildPipeline(); jet.newJob(p).join(); } finally { Jet.shutdownAll(); } }
public static void main(String[] args) throws Exception { System.setProperty("hazelcast.logging.type", "log4j"); JetConfig jetConfig = getJetConfig(); JetInstance jet = Jet.newJetInstance(jetConfig); Jet.newJetInstance(jetConfig); try { Pipeline p = Pipeline.create(); p.drawFrom(Sources.<Integer, Integer>cacheJournal(CACHE_NAME, START_FROM_OLDEST)) .map(Entry::getValue) .drainTo(Sinks.list(SINK_NAME)); jet.newJob(p); ICache<Integer, Integer> cache = jet.getCacheManager().getCache(CACHE_NAME); for (int i = 0; i < 1000; i++) { cache.put(i, i); } TimeUnit.SECONDS.sleep(3); System.out.println("Read " + jet.getList(SINK_NAME).size() + " entries from cache journal."); } finally { Jet.shutdownAll(); } }
jobConfig.setName(jobName); instance1.newJob(p, jobConfig); List<Job> jobs = instance2.getJobs(); Job trackedJob1 = jobs.get(0); Job trackedJob2 = requireNonNull(instance1.getJob(jobName)); System.out.println("Tracked job status: " + trackedJob2.getStatus()); instance1.shutdown(); instance2.shutdown();
@Before public void setup() { JetConfig cfg = new JetConfig(); cfg.getHazelcastConfig().addEventJournalConfig(new EventJournalConfig().setMapName(SOURCE_NAME)); jet = createJetMember(cfg); baosOut = new ByteArrayOutputStream(); baosErr = new ByteArrayOutputStream(); out = new PrintStream(baosOut); err = new PrintStream(baosErr); sourceMap = jet.getMap(SOURCE_NAME); IntStream.range(0, ITEM_COUNT).forEach(i -> sourceMap.put(i, i)); sinkList = jet.getList(SINK_NAME); }
@Override public void init(@Nonnull Context context) throws Exception { logger = context.jetInstance().getHazelcastInstance().getLoggingService().getLogger(ReadHdfsP.class); int totalParallelism = context.totalParallelism(); InputFormat inputFormat = jobConf.getInputFormat(); InputSplit[] splits = inputFormat.getSplits(jobConf, totalParallelism); IndexedInputSplit[] indexedInputSplits = new IndexedInputSplit[splits.length]; Arrays.setAll(indexedInputSplits, i -> new IndexedInputSplit(i, splits[i])); Address[] addrs = context.jetInstance().getCluster().getMembers() .stream().map(Member::getAddress).toArray(Address[]::new); assigned = assignSplitsToMembers(indexedInputSplits, addrs); printAssignments(assigned); }
private void runWithJet(Consumer<JetInstance> consumer) throws IOException { configureLogging(); JetInstance jet = jetClientFn.apply(getClientConfig()); try { consumer.accept(jet); } finally { jet.shutdown(); } }
/** * Creates and returns an executable job based on the supplied pipeline. * Jet will asynchronously start executing the job. */ @Nonnull default Job newJob(@Nonnull Pipeline pipeline) { return newJob(pipeline.toDag()); }
private void printResults() { final int limit = 100; System.out.format(" Top %d entries are:%n", limit); final Map<String, Long> counts = jet.getMap(COUNTS); System.out.println("/-------+---------\\"); System.out.println("| Count | Word |"); System.out.println("|-------+---------|"); counts.entrySet().stream() .sorted(comparingLong(Entry<String, Long>::getValue).reversed()) .limit(limit) .forEach(e -> System.out.format("|%6d | %-8s|%n", e.getValue(), e.getKey())); System.out.println("\\-------+---------/"); }
public static void main(String[] args) { ApplicationContext context = new GenericXmlApplicationContext("application-context.xml"); JetInstance jetInstance = (JetInstance) context.getBean("instance"); JetInstance jetClient = (JetInstance) context.getBean("client"); Pipeline pipeline = Pipeline.create(); pipeline.drawFrom(CustomSourceP.customSource()) .drainTo(Sinks.logger()); JobConfig jobConfig = new JobConfig() .addClass(XmlConfigurationSample.class) .addClass(CustomSourceP.class); jetClient.newJob(pipeline, jobConfig).join(); jetClient.shutdown(); jetInstance.shutdown(); } }
try { System.out.println("Creating " + ITEM_COUNT + " entries..."); IMap<String, Trade> sourceMap = jet.getMap(SOURCE_NAME); for (int i = 0; i < ITEM_COUNT; i++) { String ticker = "ticker-" + i; ).drainTo(Sinks.list(SINK_NAME)); System.out.println("\n\nExecuting job 1...\n"); jet.newJob(p1).join(); IList<String> sink = jet.getList(SINK_NAME); System.out.println("Sink items using predicates and projections: " + sink.subList(0, sink.size())); sink.clear(); ).drainTo(Sinks.list(SINK_NAME)); System.out.println("\n\nExecuting job 2...\n"); jet.newJob(p2).join();