@Override public void cleanup() { processorRuntime.cleanup(); }
@Override public List<Result> process(StreamlineEvent inputEvent) throws ProcessingException { if (exception != null) { throw new RuntimeException(exception); } return processEvent(inputEvent); }
@Override public void initialize(Map<String, Object> config) { initializeRuleRuntimes(config); buildStreamToRulesRuntime(); this.processAll = this.rulesProcessor.getProcessAll(); }
markWaitingSubprocess(); ProcessorMsg processorMsg = createProcessorMessage(inputEvent); shellProcess.writeProcessorMsg(processorMsg); throw new IllegalArgumentException("Command not found in shell message: " + shellMsg); setHeartbeat(); results.add(convertShellMsg(entry.getKey(), entry.getValue(), inputEvent)); throw new RuntimeException(processInfo, e); } finally { completedWaitingSubprocess();
private void initializeRuleRuntimes(Map<String, Object> config) { List<Rule> rules = rulesProcessor.getRules(); if (rules != null) { for (Rule rule: rules) { RuleRuntime ruleRuntime; Script script = null; if (ScriptType.GROOVY.equals(scriptType)) { script = createGroovyScript(rule); } else if (ScriptType.SQL.equals(scriptType)) { script = createSqlScript(rule); } else { throw new RuntimeException("Ruleruntime scriptType unsupported: " + scriptType ); } ruleRuntime = new RuleRuntime(rule, script, createActionRuntimes(rule)); rulesRuntime.add(ruleRuntime); ruleRuntime.initialize(config); } LOG.info("ruleRuntimes [{}]", rulesRuntime); } }
@Override public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { super.prepare(stormConf, context, collector); if (this.rulesProcessor == null) { throw new RuntimeException("rulesProcessor cannot be null"); } ruleProcessorRuntime = new RuleProcessorRuntime(rulesProcessor, scriptType); Map<String, Object> config = Collections.emptyMap(); ruleProcessorRuntime.initialize(config); }
@Override public void initialize(Map<String, Object> config) { command = (String[]) config.get(COMMAND); processTimeoutMills = (int) config.get(PROCESS_TIMEOUT_MILLS); Map<String, Object> processorConfig = (Map<String, Object>) config.get(PROCESS_CONFIG); ShellContext shellContext = (ShellContext) config.get(SHELL_CONTEXT); List<String> outputStreams = (List<String>) config.get(OUTPUT_STREAMS); Map<String, String> envMap = (Map<String, String>) config.get(SHELL_ENVIRONMENT); String className = (String) config.get(MULTILANG_SERIALIZER); shellProcess = new ShellProcess(command); if(className != null) shellProcess.setSerializerClassName(className); shellProcess.setEnv(envMap); //subprocesses must send their pid first thing Long subpid = shellProcess.launch(processorConfig, shellContext, outputStreams); LOG.info("Launched subprocess with pid " + subpid); LOG.info("Start checking heartbeat..."); setHeartbeat(); heartBeatExecutorService = MoreExecutors.getExitingScheduledExecutorService(new ScheduledThreadPoolExecutor(1)); heartBeatExecutorService.scheduleAtFixedRate(new HeartbeatTimerTask(this), 1, 1, TimeUnit.SECONDS); }
public void prepare(Map stormConf, TopologyContext context, final OutputCollector collector) { super.prepare(stormConf, context, collector); rand = new Random(); ShellContext shellContext = getShellContext(context); Map<String, Object> processConfigMap = new HashMap<>(); processConfigMap.put(MultiLangProcessorRuntime.COMMAND, command.split(" ")); processConfigMap.put(MultiLangProcessorRuntime.PROCESS_CONFIG, stormConf); processConfigMap.put(MultiLangProcessorRuntime.SHELL_CONTEXT, shellContext); processConfigMap.put(MultiLangProcessorRuntime.OUTPUT_STREAMS, outputStreams); processConfigMap.put(MultiLangProcessorRuntime.PROCESS_TIMEOUT_MILLS, processTimeoutInMs); processConfigMap.put(MultiLangProcessorRuntime.SHELL_ENVIRONMENT, envMap); processorRuntime = new MultiLangProcessorRuntime(); processorRuntime.initialize(processConfigMap); }
@Override protected void process(Tuple input, StreamlineEvent event) { // Input tuple is expected to be an StreamlineEvent StreamlineEvent eventWithStream = getStreamlineEventWithStream(event, input); LOG.debug("++++++++ Executing tuple [{}], StreamlineEvent [{}]", input, eventWithStream); try { for (Result result : ruleProcessorRuntime.process(eventWithStream)) { for (StreamlineEvent e : result.events) { collector.emit(result.stream, input, new Values(e)); } } } catch (ProcessingException e) { throw new RuntimeException(e); } }
private void markWaitingSubprocess() { if(!waitingOnSubprocess.get()) setHeartbeat(); waitingOnSubprocess.compareAndSet(false, true); }
void initializeWithRuleRuntimesForTesting(Map<String, Object> config, List<RuleRuntime> ruleRuntimes) { rulesRuntime = ruleRuntimes; buildStreamToRulesRuntime(); this.processAll = this.rulesProcessor.getProcessAll(); }
private Result convertShellMsg(String stream, List<ShellMsg> shellMsgList, StreamlineEvent inputEvent) { List<StreamlineEvent> streamlineEvents = new LinkedList<>(); for (ShellMsg shellMsg: shellMsgList) { streamlineEvents.add(convertShellEvent(shellMsg.getStreamlineEvent(), inputEvent)); } return new Result(stream, streamlineEvents); }
@Override protected void process(Tuple input, StreamlineEvent event) { //just need an id StreamlineEvent eventWithStream = getStreamlineEventWithStream(event, input); try { for (Result result : processorRuntime.process(eventWithStream)) { for (StreamlineEvent e : result.events) { collector.emit(result.stream, input, new Values(e)); } } } catch (ProcessingException e) { throw new RuntimeException(e); } }
private Script createSqlScript(Rule rule) { SqlEngine sqlEngine = new SqlEngine(); LOG.info("Built sqlEngine {}", sqlEngine); StormSqlExpression stormSqlExpression = createSqlExpression(rule); SqlScript sqlScript = new SqlScript(stormSqlExpression, sqlEngine); LOG.info("Built SqlScript {}", sqlScript); SqlScript.CorrelatedValuesToStreamlineEventConverter valuesConverter = new SqlScript.CorrelatedValuesToStreamlineEventConverter(sqlScript.getOutputFields()); sqlScript.setValuesConverter(valuesConverter); LOG.info("valuesConverter {}", valuesConverter); return sqlScript; }
private Script createGroovyScript(Rule rule) { LOG.info("Creating groovy execution script for rule {} ", rule); GroovyExpression groovyExpression = new GroovyExpression(rule.getCondition()); GroovyScriptEngine groovyScriptEngine = new GroovyScriptEngine(); GroovyScript<Boolean> groovyScript = createHelperGroovyScript(groovyExpression, groovyScriptEngine); GroovyScript<Collection<StreamlineEvent>> wrapper = new GroovyScript<Collection<StreamlineEvent>>(groovyExpression.asString(), groovyScriptEngine) { @Override public Collection<StreamlineEvent> evaluate(StreamlineEvent event) throws ScriptException { if (groovyScript.evaluate(event)) { return Collections.singletonList(event); } else { return Collections.emptyList(); } } }; return wrapper; }
@Override public List<Result> process(StreamlineEvent event) throws ProcessingException { List<Result> results = new ArrayList<>(); try { List<RuleRuntime> ruleRuntimes = getRulesRuntime(event); LOG.debug("Process event {}, rule runtimes {}", event, ruleRuntimes); for (RuleRuntime rr : ruleRuntimes) { boolean succeeded = false; for (StreamlineEvent result : rr.evaluate(event)) { if (result != null) { results.addAll(rr.process(result)); succeeded = true; } } if(!processAll && succeeded) break; } } catch (Exception e) { String message = String.format("Error evaluating rule processor with id: %s, error: %s", rulesProcessor.getId(), e.getMessage()); LOG.error(message, e); throw new ProcessingException(message, e); } return results; }
@Override public void prepare(Map stormConf, TopologyContext context, OutputCollector collector) { super.prepare(stormConf, context, collector); if (this.rulesProcessor == null) { throw new RuntimeException("rulesProcessor cannot be null"); } ruleProcessorRuntime = new RuleProcessorRuntime(rulesProcessor, scriptType); Map<String, Object> config = Collections.emptyMap(); if (stormConf != null) { config = new HashMap<>(); config.put(Constants.CATALOG_ROOT_URL, stormConf.get(Constants.CATALOG_ROOT_URL)); config.put(Constants.LOCAL_FILES_PATH, stormConf.get(Constants.LOCAL_FILES_PATH)); } ruleProcessorRuntime.initialize(config); }
private void processAndEmit(StreamlineEvent event, Map<String, Tuple> curGroup) throws ProcessingException { List<Result> results = ruleProcessorRuntime.process(eventWithWindowId(event)); for (Result result : results) { for (StreamlineEvent e : result.events) { // TODO: updateHeaders can be handled at ruleProcessorRuntime.process stage passing context info. // anchor parent events if such information is available if (EventCorrelationInjector.containsParentIds(e)) { Set<String> parentIds = EventCorrelationInjector.getParentIds(e); List<Tuple> parents = parentIds.stream().map(pid -> { if (!curGroup.containsKey(pid)) { throw new IllegalStateException("parents should be in grouped tuples"); } return curGroup.get(pid); }).collect(Collectors.toList()); collector.emit(result.stream, parents, new Values(updateHeaders(e, parents))); } else { // put all events in current group if there's no information collector.emit(result.stream, curGroup.values(), new Values(updateHeaders(e, curGroup.values()))); } } } }