/** * Flush will be called in synchronous way like StormBolt.execute() as Storm OutputCollector is not thread-safe */ @Override public void flush() { if (!queue.isEmpty()) { List<AlertStreamEvent> events = new ArrayList<>(); queue.drainTo(events); events.forEach((event) -> delegate.emit(Arrays.asList(event.getStreamId(), event))); LOG.info("Flushed {} alerts", events.size()); } lastFlushTime.set(System.currentTimeMillis()); }
public String streamEventToJson(AlertStreamEvent event) { Map<String, Object> jsonMap = new HashMap<String, Object>(); jsonMap.put("policyId", event.getPolicyId()); jsonMap.put("streamId", event.getStreamId()); jsonMap.put("createBy", event.getCreatedBy()); jsonMap.put("createTime", event.getCreatedTime()); // data int size = event.getData().length; List<StreamColumn> columns = event.getSchema().getColumns(); for (int i = 0; i < size; i++) { if (columns.size() < i) { // redundant check to log inconsistency LOG.error(" stream event data have different lenght compare to column definition! "); } else { jsonMap.put(columns.get(i).getName(), event.getData()[i]); } } return JsonUtils.writeValueAsString(jsonMap); }
@Override public String toString() { List<String> dataStrings = new ArrayList<>(this.getData().length); for (Object obj : this.getData()) { if (obj != null) { dataStrings.add(obj.toString()); } else { dataStrings.add(null); } } return String.format("Alert {site=%s, stream=%s,timestamp=%s,data=%s, policyId=%s, createdBy=%s, metaVersion=%s}", this.getSiteId(), this.getStreamId(), DateTimeUtil.millisecondsToHumanDateWithMilliseconds(this.getTimestamp()), this.getDataMap(), this.getPolicyId(), this.getCreatedBy(), this.getMetaVersion()); }
private static VelocityContext buildAlertContext(PolicyDefinition policyDefinition, AlertStreamEvent event) { VelocityContext context = new VelocityContext(); context.put(AlertContextFields.SITE_ID, event.getSiteId()); context.put(AlertContextFields.STREAM_ID, event.getStreamId()); context.put(AlertContextFields.ALERT_ID, event.getAlertId()); context.put(AlertContextFields.CREATED_BY, event.getCreatedBy()); context.put(AlertContextFields.CREATED_TIMESTAMP, event.getCreatedTime()); context.put(AlertContextFields.CREATED_TIME, String.format("%s %s", DateTimeUtil.millisecondsToHumanDateWithSeconds(event.getCreatedTime()), DateTimeUtil.CURRENT_TIME_ZONE.getID())); context.put(AlertContextFields.ALERT_TIMESTAMP, event.getTimestamp()); context.put(AlertContextFields.ALERT_TIME, String.format("%s %s", DateTimeUtil.millisecondsToHumanDateWithSeconds(event.getTimestamp()), DateTimeUtil.CURRENT_TIME_ZONE.getID())); context.put(AlertContextFields.ALERT_SCHEMA, event.getSchema()); context.put(AlertContextFields.ALERT_EVENT, event); context.put(AlertContextFields.POLICY_ID, policyDefinition.getName()); context.put(AlertContextFields.POLICY_DESC, policyDefinition.getDescription()); context.put(AlertContextFields.POLICY_TYPE, policyDefinition.getDefinition().getType()); context.put(AlertContextFields.POLICY_DEFINITION, policyDefinition.getDefinition().getValue()); context.put(AlertContextFields.POLICY_HANDLER, policyDefinition.getDefinition().getHandlerClass()); for (Map.Entry<String, Object> entry : event.getDataMap().entrySet()) { context.put(entry.getKey(), entry.getValue()); } return context; } }
List<AlertStreamEvent> outputEvents = checkDedup(event, new EventUniq(event.getStreamId(), event.getPolicyId(), event.getCreatedTime(), customFieldValues), stateFiledValue); if (outputEvents != null && outputEvents.size() > 0) {
DateTimeUtil.millisecondsToHumanDateWithSeconds(event.getCreatedTime()), DateTimeUtil.CURRENT_TIME_ZONE.getID())); alertContext.put(PublishConstants.ALERT_EMAIL_STREAM_ID, event.getStreamId()); alertContext.put(PublishConstants.ALERT_EMAIL_CREATOR, event.getCreatedBy()); alertContext.put(PublishConstants.ALERT_EMAIL_VERSION, Version.version);
public AlertStreamEvent(AlertStreamEvent event) { this.siteId = event.getSiteId(); this.alertId = event.getAlertId(); this.policyId = event.policyId; this.schema = event.schema; this.createdBy = event.createdBy; this.createdTime = event.createdTime; this.setTimestamp(event.getTimestamp()); this.setData(new Object[event.data.length]); System.arraycopy(event.data, 0, this.data, 0, event.data.length); this.setStreamId(event.getStreamId()); this.setMetaVersion(event.getMetaVersion()); }
@Override public void execute(Tuple input) { try { streamContext.counter().incr("receive_count"); PublishPartition partition = (PublishPartition) input.getValueByField(AlertConstants.FIELD_0); AlertStreamEvent event = (AlertStreamEvent) input.getValueByField(AlertConstants.FIELD_1); if (logEventEnabled) { LOG.info("Alert publish bolt {}/{} with partition {} received event: {}", this.getBoltId(), this.context.getThisTaskId(), partition, event); } DedupKey dedupKey = new DedupKey(event.getPolicyId(), event.getStreamId()); if (deduplicatorMap != null && deduplicatorMap.containsKey(dedupKey)) { List<AlertStreamEvent> eventList = deduplicatorMap.get(dedupKey).dedup(event); if (eventList == null || eventList.isEmpty()) { collector.ack(input); return; } event.setDuplicationChecked(true); } AlertStreamEvent filteredEvent = alertFilter.filter(event); if (filteredEvent != null) { alertPublisher.nextEvent(partition, filteredEvent); } this.collector.ack(input); streamContext.counter().incr("ack_count"); } catch (Throwable ex) { streamContext.counter().incr("fail_count"); LOG.error(ex.getMessage(), ex); collector.reportError(ex); } }
public static AlertPublishEvent createAlertPublishEvent(AlertStreamEvent event) { Preconditions.checkNotNull(event.getAlertId(), "alertId is not initialized before being published: " + event.toString()); AlertPublishEvent alertEvent = new AlertPublishEvent(); alertEvent.setAlertId(event.getAlertId()); alertEvent.setPolicyId(event.getPolicyId()); alertEvent.setAlertTimestamp(event.getCreatedTime()); alertEvent.setStreamId(event.getStreamId()); alertEvent.setCreatedBy(event.getCreatedBy()); alertEvent.setCreatedTime(event.getCreatedTime()); alertEvent.setAlertSubject(event.getSubject()); alertEvent.setAlertBody(event.getBody()); if (event.getContext() != null && !event.getContext().isEmpty()) { if (event.getContext().containsKey(SITE_ID_KEY)) { alertEvent.setSiteId(event.getContext().get(SITE_ID_KEY).toString()); } if (event.getContext().containsKey(POLICY_VALUE_KEY)) { alertEvent.setPolicyValue(event.getContext().get(POLICY_VALUE_KEY).toString()); } if (event.getContext().containsKey(APP_IDS_KEY)) { alertEvent.setAppIds((List<String>) event.getContext().get(APP_IDS_KEY)); } } alertEvent.setAlertData(event.getDataMap()); return alertEvent; }
@Test public void testSlackPublishment() throws Exception { Config config = ConfigFactory.load("application-test.conf"); AlertPublisher publisher = new AlertPublisherImpl("alertPublishBolt"); publisher.init(config, new HashMap()); List<Publishment> pubs = loadEntities("/router/publishments-slack.json", Publishment.class); publisher.onPublishChange(pubs, null, null, null); AlertStreamEvent event1 = createSeverityWithStreamDef("switch1", "testapp1", "Memory 1 inconsistency detected", "WARNING", "docId1", "ed01", "distribution switch", "us"); AlertStreamEvent event2 = createSeverityWithStreamDef("switch2", "testapp2", "Memory 2 inconsistency detected", "CRITICAL", "docId2", "ed02", "distribution switch", "us"); AlertStreamEvent event3 = createSeverityWithStreamDef("switch2", "testapp2", "Memory 3 inconsistency detected", "WARNING", "docId3", "ed02", "distribution switch", "us"); publisher.nextEvent(new PublishPartition(event1.getStreamId(), event1.getPolicyId(), pubs.get(0).getName(), pubs.get(0).getPartitionColumns()), event1); publisher.nextEvent(new PublishPartition(event2.getStreamId(), event2.getPolicyId(), pubs.get(0).getName(), pubs.get(0).getPartitionColumns()), event2); publisher.nextEvent(new PublishPartition(event3.getStreamId(), event3.getPolicyId(), pubs.get(0).getName(), pubs.get(0).getPartitionColumns()), event3); }
@SuppressWarnings("rawtypes") @Ignore @Test public void test() { Config config = ConfigFactory.load("application-test.conf"); AlertPublisher publisher = new AlertPublisherImpl("alertPublishBolt"); publisher.init(config, new HashMap()); PublishSpec spec = MetadataSerDeser.deserialize(getClass().getResourceAsStream("/testPublishSpec.json"), PublishSpec.class); publisher.onPublishChange(spec.getPublishments(), null, null, null); AlertStreamEvent event = create("testAlertStream"); publisher.nextEvent(new PublishPartition(event.getStreamId(), event.getPolicyId(), spec.getPublishments().get(0).getName(), spec.getPublishments().get(0).getPartitionColumns()), event); AlertStreamEvent event1 = create("testAlertStream"); publisher.nextEvent(new PublishPartition(event1.getStreamId(), event1.getPolicyId(), spec.getPublishments().get(0).getName(), spec.getPublishments().get(0).getPartitionColumns()), event1); }
EventUniq eventkey = new EventUniq(event.getStreamId(), event.getPolicyId(), event.getCreatedTime(), customFieldValues); LOG.info("event key: " + eventkey); LOG.info("dedup field: " + this.getDedupStateField());
@Test public void testNormal() throws Exception { Config config = ConfigFactory.load(); DedupCache dedupCache = new DedupCache(config, "testPublishment"); StreamDefinition stream = createStream(); PolicyDefinition policy = createPolicy(stream.getStreamId(), "testPolicy"); String[] states = new String[] {"OPEN", "WARN", "CLOSE"}; Random random = new Random(); for (int i = 0; i < 20; i++) { AlertStreamEvent event = createEvent(stream, policy, new Object[] { System.currentTimeMillis(), "host1", "testPolicy-host1-01", states[random.nextInt(3)], 0, 0 }); HashMap<String, String> dedupFieldValues = new HashMap<String, String>(); dedupFieldValues.put("alertKey", (String) event.getData()[event.getSchema().getColumnIndex("alertKey")]); List<AlertStreamEvent> result = dedupCache.dedup(event, new EventUniq(event.getStreamId(), event.getPolicyId(), event.getCreatedTime(), dedupFieldValues), "state", (String) event.getData()[event.getSchema().getColumnIndex("state")], "closed"); System.out.println((i + 1) + " >>>> " + ToStringBuilder.reflectionToString(result)); } Assert.assertTrue(true); }
event.setStreamId(originalEvent.getStreamId()); event.setSchema(originalEvent.getSchema()); event.setPolicyId(originalEvent.getPolicyId());
Assert.assertEquals("joinedStream", alerts.get(0).getStreamId()); Assert.assertEquals("cpu", alerts.get(0).getData()[1]);