protected void replaceAlertInMetaAlert(Document metaAlert, Document alert) { boolean metaAlertUpdated = removeAlertsFromMetaAlert(metaAlert, Collections.singleton(alert.getGuid())); if (metaAlertUpdated) { addAlertsToMetaAlert(metaAlert, Collections.singleton(alert)); } } }
protected boolean replaceAlertInMetaAlert(Document metaAlert, Document alert) { boolean metaAlertUpdated = removeAlertsFromMetaAlert(metaAlert, Collections.singleton(alert.getGuid())); if (metaAlertUpdated) { addAlertsToMetaAlert(metaAlert, Collections.singleton(alert)); } return metaAlertUpdated; }
/** * Adds the provided alerts to a given metaalert. * @param metaAlert The metaalert to be given new children. * @param alerts The alerts to be added as children * @return True if metaalert is modified, false otherwise. */ protected boolean addAlertsToMetaAlert(Document metaAlert, Iterable<Document> alerts) { boolean alertAdded = false; @SuppressWarnings("unchecked") List<Map<String, Object>> currentAlerts = (List<Map<String, Object>>) metaAlert.getDocument() .get(MetaAlertConstants.ALERT_FIELD); if (currentAlerts == null) { currentAlerts = new ArrayList<>(); metaAlert.getDocument().put(MetaAlertConstants.ALERT_FIELD, currentAlerts); } Set<String> currentAlertGuids = currentAlerts.stream().map(currentAlert -> (String) currentAlert.get(GUID)).collect(Collectors.toSet()); for (Document alert : alerts) { String alertGuid = alert.getGuid(); // Only add an alert if it isn't already in the meta alert if (!currentAlertGuids.contains(alertGuid)) { currentAlerts.add(alert.getDocument()); alertAdded = true; } } return alertAdded; }
/** * Copy constructor * @param other The document to be copied. */ public Document(Document other) { this(new HashMap<>(other.getDocument()), other.getGuid(), other.getSensorType(), other.getTimestamp(), other.getDocumentID().orElse(null)); }
protected String getIndexName(Document update, String indexPostFix) throws IOException { return findIndexNameByGUID(update.getGuid(), update.getSensorType()) .orElse(ElasticsearchUtils.getIndexName(update.getSensorType(), indexPostFix, null)); }
private DocumentContainer removeCommentFromAlert(IndexDao indexDao, CommentAddRemoveRequest request, Document latest) { DocumentContainer container; try { Document document = indexDao.removeCommentFromAlert(request, latest); container = new DocumentContainer(document); LOG.debug("Removed comment from alert; indexDao={}, guid={}, sensorType={}, document={}", ClassUtils.getShortClassName(indexDao.getClass()), document.getGuid(), document.getSensorType(), document); } catch (Throwable e) { container = new DocumentContainer(e); LOG.error("Unable to remove comment from alert; indexDao={}, error={}", ClassUtils.getShortClassName(indexDao.getClass()), ExceptionUtils.getRootCauseMessage(e)); } return container; }
private DocumentContainer addCommentToAlert(IndexDao indexDao, CommentAddRemoveRequest request, Document latest) { DocumentContainer container; try { Document document = indexDao.addCommentToAlert(request, latest); container = new DocumentContainer(document); LOG.debug("Added comment to alert; indexDao={}, guid={}, sensorType={}, document={}", ClassUtils.getShortClassName(indexDao.getClass()), document.getGuid(), document.getSensorType(), document); } catch (Throwable e) { container = new DocumentContainer(e); LOG.error("Unable to add comment to alert; indexDao={}, error={}", ClassUtils.getShortClassName(indexDao.getClass()), ExceptionUtils.getRootCauseMessage(e)); } return container; }
/** * Given a Metaalert and a status change, builds the set of updates to be run. * @param metaAlert The metaalert to have status changed * @param alerts The alerts to change status for * @param status The status to change to * @return The updates to be run */ protected Map<Document, Optional<String>> buildStatusChangeUpdates(Document metaAlert, Iterable<Document> alerts, MetaAlertStatus status) { metaAlert.getDocument().put(MetaAlertConstants.STATUS_FIELD, status.getStatusString()); Map<Document, Optional<String>> updates = new HashMap<>(); updates.put(metaAlert, Optional.of(config.getMetaAlertIndex())); for (Document alert : alerts) { boolean metaAlertAdded = false; boolean metaAlertRemoved = false; // If we're making it active add add the meta alert guid for every alert. if (MetaAlertStatus.ACTIVE.equals(status)) { metaAlertAdded = addMetaAlertToAlert(metaAlert.getGuid(), alert); } // If we're making it inactive, remove the meta alert guid from every alert. if (MetaAlertStatus.INACTIVE.equals(status)) { metaAlertRemoved = removeMetaAlertFromAlert(metaAlert.getGuid(), alert); } if (metaAlertAdded || metaAlertRemoved) { updates.put(alert, Optional.empty()); } } return updates; }
/** * Builds the updates to be run based on a given metaalert and a set of new alerts for the it. * @param metaAlert The base metaalert we're building updates for * @param alerts The alerts being added * @return The set of resulting updates. */ protected Map<Document, Optional<String>> buildAddAlertToMetaAlertUpdates(Document metaAlert, Iterable<Document> alerts) { Map<Document, Optional<String>> updates = new HashMap<>(); boolean metaAlertUpdated = addAlertsToMetaAlert(metaAlert, alerts); if (metaAlertUpdated) { MetaScores .calculateMetaScores(metaAlert, config.getThreatTriageField(), config.getThreatSort()); updates.put(metaAlert, Optional.of(config.getMetaAlertIndex())); for (Document alert : alerts) { if (addMetaAlertToAlert(metaAlert.getGuid(), alert)) { updates.put(alert, Optional.empty()); } } } return updates; }
SearchResponse response = getMetaAlertsForAlert(update.getGuid()); Collection<Document> metaAlerts = response .getResults()
protected Put buildPut(Document update) throws IOException { Key k = new Key(update.getGuid(), update.getSensorType()); Put put = new Put(Key.toBytes(k)); long ts = update.getTimestamp() == null || update.getTimestamp() == 0 ? System.currentTimeMillis() : update.getTimestamp(); byte[] columnQualifier = Bytes.toBytes(ts); byte[] doc = JSONUtils.INSTANCE.toJSONPretty(update.getDocument()); put.addColumn(cf, columnQualifier, doc); return put; }
private IndexRequest createRequest(D document, String index) { if(document.getTimestamp() == null) { throw new IllegalArgumentException("Document must contain the timestamp"); } // if updating an existing document, the doc ID should be defined. // if creating a new document, set the doc ID to null to allow Elasticsearch to generate one. String docId = document.getDocumentID().orElse(null); if(LOG.isDebugEnabled() && document.getDocumentID().isPresent()) { LOG.debug("Updating existing document with known doc ID; docID={}, guid={}, sensorType={}", docId, document.getGuid(), document.getSensorType()); } else if(LOG.isDebugEnabled()) { LOG.debug("Creating a new document, doc ID not yet known; guid={}, sensorType={}", document.getGuid(), document.getSensorType()); } return new IndexRequest() .source(document.getDocument()) .type(document.getSensorType() + "_doc") .index(index) .id(docId) .index(index) .timestamp(document.getTimestamp().toString()); }
@Override public Document update(Document update, Optional<String> index) throws IOException { for (Map.Entry<String, List<String>> kv : BACKING_STORE.entrySet()) { if (kv.getKey().startsWith(update.getSensorType())) { for (Iterator<String> it = kv.getValue().iterator(); it.hasNext(); ) { String doc = it.next(); Map<String, Object> docParsed = parse(doc); if (docParsed.getOrDefault(Constants.GUID, "").equals(update.getGuid())) { it.remove(); } } kv.getValue().add(JSONUtils.INSTANCE.toJSON(update.getDocument(), true)); } } return update; }
@Test public void get_all_latest_guid() throws Exception { List<GetRequest> request = JSONUtils.INSTANCE.load(getAllLatestQuery, new JSONUtils.ReferenceSupplier<List<GetRequest>>(){}); Map<String, Document> docs = new HashMap<>(); for(Document doc : getIndexDao().getAllLatest(request)) { docs.put(doc.getGuid(), doc); } Assert.assertEquals(2, docs.size()); Assert.assertTrue(docs.keySet().contains("bro_1")); Assert.assertTrue(docs.keySet().contains("snort_2")); Assert.assertEquals("bro", docs.get("bro_1").getDocument().get(getSourceTypeField())); Assert.assertEquals("snort", docs.get("snort_2").getDocument().get(getSourceTypeField())); }
@Test public void testPatchDocumentThatHasComment() throws Exception { Document document = createAndIndexDocument("testPatchDocumentWithComment"); // comment on the document String commentText = "New Comment"; String commentUser = "test_user"; long commentTimestamp = 152630493050L; Document withComment = addAlertComment(document.getGuid(), commentText, commentUser, commentTimestamp); // create a patch List<Map<String, Object>> patches = new ArrayList<>(); Map<String, Object> patch = new HashMap<>(); patch.put("op", "add"); patch.put("path", "/project"); patch.put("value", "metron"); patches.add(patch); PatchRequest pr = new PatchRequest(); pr.setGuid(withComment.getGuid()); pr.setIndex(SENSOR_NAME); pr.setSensorType(SENSOR_NAME); pr.setPatch(patches); // patch the document that has been commented on Document patched = getDao().patch(getDao(), pr, Optional.of(withComment.getTimestamp())); Assert.assertEquals("metron", patched.getDocument().get("project")); // ensure the patch was made on the indexed document Document indexed = findUpdatedDoc(patched.getDocument(), patched.getGuid(), SENSOR_NAME); Assert.assertEquals("metron", indexed.getDocument().get("project")); }
@Test public void testWrite() throws Exception { // create some documents to write List<Document> documents = new ArrayList<>(); for(int i=0; i<10; i++) { Document document = Document.fromJSON(createMessage()); documents.add(document); } // write the documents for(Document doc: documents) { writer.addDocument(doc, "bro_index"); } writer.write(); // ensure the documents were written for(Document expected: documents) { Document actual = retrieveDao.getLatest(expected.getGuid(), expected.getSensorType()); assertNotNull("No document found", actual); assertEquals(expected.getGuid(), actual.getGuid()); assertEquals(expected.getSensorType(), actual.getSensorType()); assertEquals(expected.getDocument(), actual.getDocument()); assertTrue(actual.getDocumentID().isPresent()); // the document ID and GUID should not be the same, since the document ID was auto-generated assertNotEquals(actual.getDocument(), actual.getGuid()); } }
@Test public void testAddComment() throws Exception { Document document = createAndIndexDocument("testAddCommentAndPatch"); // comment on the document String commentText = "New Comment"; String commentUser = "test_user"; long commentTimestamp = 152630493050L; Document withComment = addAlertComment(document.getGuid(), commentText, commentUser, commentTimestamp); { // validate that the comment was made on the returned document List<AlertComment> comments = getComments(withComment); Assert.assertEquals(1, comments.size()); Assert.assertEquals(commentText, comments.get(0).getComment()); Assert.assertEquals(commentUser, comments.get(0).getUsername()); Assert.assertEquals(commentTimestamp, comments.get(0).getTimestamp()); } { // validate that the comment was made on the indexed document Document indexed = findUpdatedDoc(withComment.getDocument(), withComment.getGuid(), SENSOR_NAME); List<AlertComment> comments = getComments(indexed); Assert.assertEquals(1, comments.size()); Assert.assertEquals(commentText, comments.get(0).getComment()); Assert.assertEquals(commentUser, comments.get(0).getUsername()); Assert.assertEquals(commentTimestamp, comments.get(0).getTimestamp()); } }
@Test public void testRemoveComments() throws Exception { String guid = "testRemoveComments"; createAndIndexDocument(guid); // add a comment on the document Document withComments = addAlertComment(guid, "comment", "user1", 1526401584951L); Assert.assertEquals(1, getComments(withComments).size()); // ensure the comment was added to the document in the index Document indexedWithComments = findUpdatedDoc(withComments.getDocument(), withComments.getGuid(), withComments.getSensorType()); Assert.assertEquals(1, getComments(indexedWithComments).size()); // remove a comment from the document AlertComment toRemove = getComments(withComments).get(0); Document noComments = removeAlertComment(guid, toRemove.getComment(), toRemove.getUsername(), toRemove.getTimestamp()); Assert.assertEquals(0, getComments(noComments).size()); // ensure the comment was removed from the index Document indexedNoComments = findUpdatedDoc(noComments.getDocument(), withComments.getGuid(), withComments.getSensorType()); Assert.assertEquals(0, getComments(indexedNoComments).size()); }