@Override public synchronized AlertStreamEvent filter(AlertStreamEvent event) { Preconditions.checkArgument(this.policyDefinitionRepository.containsKey(event.getPolicyId()), "Unknown policyId " + event.getPolicyId()); PolicyDefinition policyDefinition = this.policyDefinitionRepository.get(event.getPolicyId()); StringWriter bodyWriter = new StringWriter(); StringWriter subjectWriter = new StringWriter(); try { VelocityContext alertContext = buildAlertContext(policyDefinition, event); Template template = engine.getTemplate(getAlertBodyTemplateName(event.getPolicyId())); template.merge(alertContext, bodyWriter); event.setBody(bodyWriter.toString()); template = engine.getTemplate(getAlertSubjectTemplateName(event.getPolicyId())); template.merge(alertContext, subjectWriter); event.setSubject(subjectWriter.toString()); } finally { try { bodyWriter.close(); } catch (IOException e) { LOG.warn(e.getMessage(), e); } try { subjectWriter.close(); } catch (IOException e) { LOG.warn(e.getMessage(), e); } } return event; }
private String getAlertBody(AlertStreamEvent event) { if (event.getBody() == null) { return String.format("Alert policy \"%s\" was triggered: %s", event.getPolicyId(), generateAlertDataDesc(event)); } else { return event.getBody(); } }
public String streamEventToJson(AlertStreamEvent event) { Map<String, Object> jsonMap = new HashMap<String, Object>(); jsonMap.put("policyId", event.getPolicyId()); jsonMap.put("streamId", event.getStreamId()); jsonMap.put("createBy", event.getCreatedBy()); jsonMap.put("createTime", event.getCreatedTime()); // data int size = event.getData().length; List<StreamColumn> columns = event.getSchema().getColumns(); for (int i = 0; i < size; i++) { if (columns.size() < i) { // redundant check to log inconsistency LOG.error(" stream event data have different lenght compare to column definition! "); } else { jsonMap.put(columns.get(i).getName(), event.getData()[i]); } } return JsonUtils.writeValueAsString(jsonMap); }
@Override public String toString() { List<String> dataStrings = new ArrayList<>(this.getData().length); for (Object obj : this.getData()) { if (obj != null) { dataStrings.add(obj.toString()); } else { dataStrings.add(null); } } return String.format("Alert {site=%s, stream=%s,timestamp=%s,data=%s, policyId=%s, createdBy=%s, metaVersion=%s}", this.getSiteId(), this.getStreamId(), DateTimeUtil.millisecondsToHumanDateWithMilliseconds(this.getTimestamp()), this.getDataMap(), this.getPolicyId(), this.getCreatedBy(), this.getMetaVersion()); }
alertContext.put(PublishConstants.ALERT_EMAIL_POLICY_ID, event.getPolicyId()); alertContext.put(PublishConstants.ALERT_EMAIL_ALERT_ID, event.getAlertId()); alertContext.put(PublishConstants.ALERT_EMAIL_ALERT_DATA, event.getDataMap().toString()); String.format("%s/#/site/%s/alert/detail/%s?timestamp=%s", rootUrl, event.getSiteId(), URIUtil.encodeQuery(event.getAlertId(), "UTF-8"), event.getTimestamp())); alertContext.put(PublishConstants.ALERT_EMAIL_POLICY_DETAIL_URL, String.format("%s/#/site/%s/policy/detail/%s", rootUrl, event.getSiteId(), URIUtil.encodeQuery(event.getPolicyId(), "UTF-8"))); } catch (URIException e) { LOG.warn(e.getMessage(), e); String.format("%s/#/site/%s/alert/detail/%s?timestamp=%s", rootUrl, event.getSiteId(), event.getAlertId(), event.getTimestamp())); alertContext.put(PublishConstants.ALERT_EMAIL_POLICY_DETAIL_URL, String.format("%s/#/site/%s/policy/detail/%s", rootUrl, event.getSiteId(), event.getPolicyId()));
event.getPolicyId(), event.getCreatedTime(), customFieldValues), stateFiledValue); if (outputEvents != null && outputEvents.size() > 0) { return outputEvents;
/** * TODO: Refactor wrapAlertPublishEvent into alertTemplateEngine and remove extraData from AlertStreamEvent. */ @Override public AlertStreamEvent filter(AlertStreamEvent event) { event.ensureAlertId(); Map<String, Object> extraData = new HashMap<>(); List<String> appIds = new ArrayList<>(); if (alertPublisherBolt.policyDefinitionMap == null || alertPublisherBolt.streamDefinitionMap == null) { LOG.warn("policyDefinitions or streamDefinitions in publisher bolt have not been initialized"); } else { PolicyDefinition policyDefinition = alertPublisherBolt.policyDefinitionMap.get(event.getPolicyId()); if (alertPublisherBolt.policyDefinitionMap != null && policyDefinition != null) { for (String inputStreamId : policyDefinition.getInputStreams()) { StreamDefinition sd = alertPublisherBolt.streamDefinitionMap.get(inputStreamId); if (sd != null) { extraData.put(AlertPublishEvent.SITE_ID_KEY, sd.getSiteId()); appIds.add(sd.getStreamSource()); } } extraData.put(AlertPublishEvent.APP_IDS_KEY, appIds); extraData.put(AlertPublishEvent.POLICY_VALUE_KEY, policyDefinition.getDefinition().getValue()); event.setSeverity(policyDefinition.getAlertSeverity()); event.setCategory(policyDefinition.getAlertCategory()); } event.setContext(extraData); } return event; } }
public AlertEntity convertAlertEvent(AlertStreamEvent event) { Preconditions.checkNotNull(event.getAlertId(), "alertId is not initialized before being published: " + event.toString()); AlertEntity alertEvent = new AlertEntity(); Map<String, String> tags = new HashMap<>(); tags.put(POLICY_ID_KEY, event.getPolicyId()); tags.put(ALERT_ID_KEY, event.getAlertId()); tags.put(ALERT_CATEGORY, event.getCategory()); tags.put(ALERT_SEVERITY, event.getSeverity().toString()); String host = event.getDataMap().getOrDefault("host", "null").toString(); String hostname = event.getDataMap().getOrDefault("hostname", "null").toString(); if (host != "null") { tags.put(ALERT_HOST, host); } else { tags.put(ALERT_HOST, hostname); } if (event.getContext() != null && !event.getContext().isEmpty()) { tags.put(SITE_ID_KEY, event.getContext().get(SITE_ID_KEY).toString()); alertEvent.setPolicyValue(event.getContext().get(POLICY_VALUE_KEY).toString()); alertEvent.setAppIds((List<String>) event.getContext().get(APP_IDS_KEY)); } alertEvent.setTimestamp(event.getCreatedTime()); alertEvent.setAlertData(event.getDataMap()); alertEvent.setAlertSubject(event.getSubject()); alertEvent.setAlertBody(event.getBody()); alertEvent.setTags(tags); return alertEvent; }
@Override public void emit(AlertStreamEvent event) { if (event == null) { return; } event.ensureAlertId(); Set<PublishPartition> clonedPublishPartitions = new HashSet<>(publishPartitions); for (PublishPartition publishPartition : clonedPublishPartitions) { // skip the publish partition which is not belong to this policy and also check streamId PublishPartition cloned = publishPartition.clone(); Optional.ofNullable(event) .filter(x -> x != null && x.getSchema() != null && cloned.getPolicyId().equalsIgnoreCase(x.getPolicyId()) && (cloned.getStreamId().equalsIgnoreCase(x.getSchema().getStreamId()) || cloned.getStreamId().equalsIgnoreCase(Publishment.STREAM_NAME_DEFAULT))) .ifPresent(x -> { cloned.getColumns().stream() .filter(y -> event.getSchema().getColumnIndex(y) >= 0 && event.getSchema().getColumnIndex(y) < event.getSchema().getColumns().size()) .map(y -> event.getData()[event.getSchema().getColumnIndex(y)]) .filter(y -> y != null) .forEach(y -> cloned.getColumnValues().add(y)); synchronized (outputLock) { streamContext.counter().incr("alert_count"); delegate.emit(Arrays.asList(cloned, event)); } }); } }
@Override public void execute(Tuple input) { try { streamContext.counter().incr("receive_count"); PublishPartition partition = (PublishPartition) input.getValueByField(AlertConstants.FIELD_0); AlertStreamEvent event = (AlertStreamEvent) input.getValueByField(AlertConstants.FIELD_1); if (logEventEnabled) { LOG.info("Alert publish bolt {}/{} with partition {} received event: {}", this.getBoltId(), this.context.getThisTaskId(), partition, event); } DedupKey dedupKey = new DedupKey(event.getPolicyId(), event.getStreamId()); if (deduplicatorMap != null && deduplicatorMap.containsKey(dedupKey)) { List<AlertStreamEvent> eventList = deduplicatorMap.get(dedupKey).dedup(event); if (eventList == null || eventList.isEmpty()) { collector.ack(input); return; } event.setDuplicationChecked(true); } AlertStreamEvent filteredEvent = alertFilter.filter(event); if (filteredEvent != null) { alertPublisher.nextEvent(partition, filteredEvent); } this.collector.ack(input); streamContext.counter().incr("ack_count"); } catch (Throwable ex) { streamContext.counter().incr("fail_count"); LOG.error(ex.getMessage(), ex); collector.reportError(ex); } }
public static AlertPublishEvent createAlertPublishEvent(AlertStreamEvent event) { Preconditions.checkNotNull(event.getAlertId(), "alertId is not initialized before being published: " + event.toString()); AlertPublishEvent alertEvent = new AlertPublishEvent(); alertEvent.setAlertId(event.getAlertId()); alertEvent.setPolicyId(event.getPolicyId()); alertEvent.setAlertTimestamp(event.getCreatedTime()); alertEvent.setStreamId(event.getStreamId()); alertEvent.setCreatedBy(event.getCreatedBy()); alertEvent.setCreatedTime(event.getCreatedTime()); alertEvent.setAlertSubject(event.getSubject()); alertEvent.setAlertBody(event.getBody()); if (event.getContext() != null && !event.getContext().isEmpty()) { if (event.getContext().containsKey(SITE_ID_KEY)) { alertEvent.setSiteId(event.getContext().get(SITE_ID_KEY).toString()); } if (event.getContext().containsKey(POLICY_VALUE_KEY)) { alertEvent.setPolicyValue(event.getContext().get(POLICY_VALUE_KEY).toString()); } if (event.getContext().containsKey(APP_IDS_KEY)) { alertEvent.setAppIds((List<String>) event.getContext().get(APP_IDS_KEY)); } } alertEvent.setAlertData(event.getDataMap()); return alertEvent; }
@Test public void testSlackPublishment() throws Exception { Config config = ConfigFactory.load("application-test.conf"); AlertPublisher publisher = new AlertPublisherImpl("alertPublishBolt"); publisher.init(config, new HashMap()); List<Publishment> pubs = loadEntities("/router/publishments-slack.json", Publishment.class); publisher.onPublishChange(pubs, null, null, null); AlertStreamEvent event1 = createSeverityWithStreamDef("switch1", "testapp1", "Memory 1 inconsistency detected", "WARNING", "docId1", "ed01", "distribution switch", "us"); AlertStreamEvent event2 = createSeverityWithStreamDef("switch2", "testapp2", "Memory 2 inconsistency detected", "CRITICAL", "docId2", "ed02", "distribution switch", "us"); AlertStreamEvent event3 = createSeverityWithStreamDef("switch2", "testapp2", "Memory 3 inconsistency detected", "WARNING", "docId3", "ed02", "distribution switch", "us"); publisher.nextEvent(new PublishPartition(event1.getStreamId(), event1.getPolicyId(), pubs.get(0).getName(), pubs.get(0).getPartitionColumns()), event1); publisher.nextEvent(new PublishPartition(event2.getStreamId(), event2.getPolicyId(), pubs.get(0).getName(), pubs.get(0).getPartitionColumns()), event2); publisher.nextEvent(new PublishPartition(event3.getStreamId(), event3.getPolicyId(), pubs.get(0).getName(), pubs.get(0).getPartitionColumns()), event3); }
@SuppressWarnings("rawtypes") @Ignore @Test public void test() { Config config = ConfigFactory.load("application-test.conf"); AlertPublisher publisher = new AlertPublisherImpl("alertPublishBolt"); publisher.init(config, new HashMap()); PublishSpec spec = MetadataSerDeser.deserialize(getClass().getResourceAsStream("/testPublishSpec.json"), PublishSpec.class); publisher.onPublishChange(spec.getPublishments(), null, null, null); AlertStreamEvent event = create("testAlertStream"); publisher.nextEvent(new PublishPartition(event.getStreamId(), event.getPolicyId(), spec.getPublishments().get(0).getName(), spec.getPublishments().get(0).getPartitionColumns()), event); AlertStreamEvent event1 = create("testAlertStream"); publisher.nextEvent(new PublishPartition(event1.getStreamId(), event1.getPolicyId(), spec.getPublishments().get(0).getName(), spec.getPublishments().get(0).getPartitionColumns()), event1); }
EventUniq eventkey = new EventUniq(event.getStreamId(), event.getPolicyId(), event.getCreatedTime(), customFieldValues); LOG.info("event key: " + eventkey); LOG.info("dedup field: " + this.getDedupStateField());
@Test public void testNormal() throws Exception { Config config = ConfigFactory.load(); DedupCache dedupCache = new DedupCache(config, "testPublishment"); StreamDefinition stream = createStream(); PolicyDefinition policy = createPolicy(stream.getStreamId(), "testPolicy"); String[] states = new String[] {"OPEN", "WARN", "CLOSE"}; Random random = new Random(); for (int i = 0; i < 20; i++) { AlertStreamEvent event = createEvent(stream, policy, new Object[] { System.currentTimeMillis(), "host1", "testPolicy-host1-01", states[random.nextInt(3)], 0, 0 }); HashMap<String, String> dedupFieldValues = new HashMap<String, String>(); dedupFieldValues.put("alertKey", (String) event.getData()[event.getSchema().getColumnIndex("alertKey")]); List<AlertStreamEvent> result = dedupCache.dedup(event, new EventUniq(event.getStreamId(), event.getPolicyId(), event.getCreatedTime(), dedupFieldValues), "state", (String) event.getData()[event.getSchema().getColumnIndex("state")], "closed"); System.out.println((i + 1) + " >>>> " + ToStringBuilder.reflectionToString(result)); } Assert.assertTrue(true); }
event.setStreamId(originalEvent.getStreamId()); event.setSchema(originalEvent.getSchema()); event.setPolicyId(originalEvent.getPolicyId()); event.setCreatedTime(originalEvent.getCreatedTime()); event.setCreatedBy(originalEvent.getCreatedBy());