/** * Copy constructor * @param other The document to be copied. */ public Document(Document other) { this(new HashMap<>(other.getDocument()), other.getGuid(), other.getSensorType(), other.getTimestamp(), other.getDocumentID().orElse(null)); }
protected List<Map<String, Object>> getRawMaps(List<Document> documents) { List<Map<String, Object>> rawMaps = new ArrayList<>(); for (Document document : documents) { rawMaps.add(document.getDocument()); } return rawMaps; }
@Override @SuppressWarnings("unchecked") public Document addCommentToAlert(CommentAddRemoveRequest request, Document latest) throws IOException { if (latest == null || latest.getDocument() == null) { throw new IOException(String.format("Unable to add comment. Document with guid %s cannot be found.", request.getGuid())); } List<Map<String, Object>> commentsField = (List<Map<String, Object>>) latest.getDocument() .getOrDefault(COMMENTS_FIELD, new ArrayList<>()); List<Map<String, Object>> originalComments = new ArrayList<>(commentsField); originalComments.add( new AlertComment(request.getComment(), request.getUsername(), request.getTimestamp()) .asMap()); Document newVersion = new Document(latest); newVersion.getDocument().put(COMMENTS_FIELD, originalComments); return update(newVersion, Optional.empty()); }
/** * Return the latest version of a document given a GetRequest. * @param request The GetRequest which indicates the GUID and sensor type. * @return Optionally the document (dependent upon existence in the index). * @throws IOException If an error occurs while retrieving the document. */ default Optional<Map<String, Object>> getLatestResult(GetRequest request) throws IOException { Document ret = getLatest(request.getGuid(), request.getSensorType()); if (ret == null) { return Optional.empty(); } else { return Optional.ofNullable(ret.getDocument()); } } }
public static SearchResult getSearchResult(SolrDocument solrDocument, List<String> fields, Function<String, String> indexSupplier) { SearchResult searchResult = new SearchResult(); searchResult.setId((String) solrDocument.getFieldValue(Constants.GUID)); searchResult.setIndex(indexSupplier.apply((String) solrDocument.getFieldValue(Constants.SENSOR_TYPE))); Map<String, Object> docSource = toDocument(solrDocument).getDocument(); final Map<String, Object> source = new HashMap<>(); if (fields != null) { fields.forEach(field -> source.put(field, docSource.get(field))); } else { source.putAll(docSource); } searchResult.setSource(source); return searchResult; }
private List<AlertComment> getComments(Document withComment) { List<Map<String, Object>> commentsField = List.class.cast(withComment.getDocument().get(COMMENTS_FIELD)); List<AlertComment> comments = new ArrayList<>(); if(commentsField != null) { comments = commentsField .stream() .map(map -> new AlertComment(map)) .collect(Collectors.toList()); } return comments; }
protected Put buildPut(Document update) throws IOException { Key k = new Key(update.getGuid(), update.getSensorType()); Put put = new Put(Key.toBytes(k)); long ts = update.getTimestamp() == null || update.getTimestamp() == 0 ? System.currentTimeMillis() : update.getTimestamp(); byte[] columnQualifier = Bytes.toBytes(ts); byte[] doc = JSONUtils.INSTANCE.toJSONPretty(update.getDocument()); put.addColumn(cf, columnQualifier, doc); return put; }
@Override public Document update(Document update, Optional<String> index) throws IOException { for (Map.Entry<String, List<String>> kv : BACKING_STORE.entrySet()) { if (kv.getKey().startsWith(update.getSensorType())) { for (Iterator<String> it = kv.getValue().iterator(); it.hasNext(); ) { String doc = it.next(); Map<String, Object> docParsed = parse(doc); if (docParsed.getOrDefault(Constants.GUID, "").equals(update.getGuid())) { it.remove(); } } kv.getValue().add(JSONUtils.INSTANCE.toJSON(update.getDocument(), true)); } } return update; }
protected void findUpdatedDoc(Map<String, Object> message0, String guid, String sensorType) throws InterruptedException, IOException, OriginalNotFoundException { commit(); for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) { Document doc = metaDao.getLatest(guid, sensorType); // Change the underlying document alerts lists to sets to avoid ordering issues. convertAlertsFieldToSet(doc.getDocument()); convertAlertsFieldToSet(message0); if (doc.getDocument() != null && message0.equals(doc.getDocument())) { convertAlertsFieldToList(doc.getDocument()); convertAlertsFieldToList(message0); return; } } throw new OriginalNotFoundException( "Count not find " + guid + " after " + MAX_RETRIES + " tries"); }
@Test public void testAddMetaAlertToAlertEmpty() { Map<String, Object> alertFields = new HashMap<>(); alertFields.put(METAALERT_FIELD, new ArrayList<>()); Document alert = new Document(alertFields, "alert", "test", 0L); Document expected = new Document(new HashMap<>(), "alert", "test", 0L); List<String> expectedMetaAlertGuids = new ArrayList<>(); expectedMetaAlertGuids.add("metaalert1"); expected.getDocument().put(METAALERT_FIELD, expectedMetaAlertGuids); boolean actual = dao.addMetaAlertToAlert("metaalert1", alert); assertTrue(actual); assertEquals(expected, alert); }
@Test public void testRemoveMetaAlertFromAlertSuccess() { List<String> metaAlertGuids = new ArrayList<>(); metaAlertGuids.add("metaalert1"); metaAlertGuids.add("metaalert2"); Map<String, Object> alertFields = new HashMap<>(); alertFields.put(METAALERT_FIELD, metaAlertGuids); Document alert = new Document(alertFields, "alert", "test", 0L); Document expected = new Document(new HashMap<>(), "alert", "test", 0L); List<String> expectedMetaAlertGuids = new ArrayList<>(); expectedMetaAlertGuids.add("metaalert2"); expected.getDocument().put(METAALERT_FIELD, expectedMetaAlertGuids); boolean actual = dao.removeMetaAlertFromAlert("metaalert1", alert); assertTrue(actual); assertEquals(expected, alert); }
@Test public void testAddMetaAlertToAlertNonEmpty() { List<String> metaAlertGuids = new ArrayList<>(); metaAlertGuids.add("metaalert1"); Map<String, Object> alertFields = new HashMap<>(); alertFields.put(METAALERT_FIELD, metaAlertGuids); Document alert = new Document(alertFields, "alert", "test", 0L); Document expected = new Document(new HashMap<>(), "alert", "test", 0L); List<String> expectedMetaAlertGuids = new ArrayList<>(); expectedMetaAlertGuids.add("metaalert1"); expectedMetaAlertGuids.add("metaalert2"); expected.getDocument().put(METAALERT_FIELD, expectedMetaAlertGuids); boolean actual = dao.addMetaAlertToAlert("metaalert2", alert); assertTrue(actual); assertEquals(expected, alert); }
protected Document findUpdatedDoc(Map<String, Object> message0, String guid, String sensorType) throws InterruptedException, IOException, OriginalNotFoundException { for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) { Document doc = getDao().getLatest(guid, sensorType); if (doc != null && message0.equals(doc.getDocument())) { return doc; } if (t == MAX_RETRIES -1) { MapUtils.debugPrint(System.out, "Expected", message0); MapUtils.debugPrint(System.out, "actual", doc.getDocument()); } } throw new OriginalNotFoundException("Count not find " + guid + " after " + MAX_RETRIES + " tries"); }
@Test public void shouldGetLatestWithInvalidTimestamp() throws Exception { // Load alert Document alert = buildAlerts(1).get(0); hbaseDao.update(alert, Optional.empty()); Document actualDocument = hbaseDao.getLatest("message_0", SENSOR_TYPE); Assert.assertEquals(alert, actualDocument); alert.getDocument().put("field", "value"); alert.setTimestamp(0L); hbaseDao.update(alert, Optional.empty()); actualDocument = hbaseDao.getLatest("message_0", SENSOR_TYPE); Assert.assertEquals(alert.getDocument(), actualDocument.getDocument()); }
@Test public void get_all_latest_guid() throws Exception { List<GetRequest> request = JSONUtils.INSTANCE.load(getAllLatestQuery, new JSONUtils.ReferenceSupplier<List<GetRequest>>(){}); Map<String, Document> docs = new HashMap<>(); for(Document doc : getIndexDao().getAllLatest(request)) { docs.put(doc.getGuid(), doc); } Assert.assertEquals(2, docs.size()); Assert.assertTrue(docs.keySet().contains("bro_1")); Assert.assertTrue(docs.keySet().contains("snort_2")); Assert.assertEquals("bro", docs.get("bro_1").getDocument().get(getSourceTypeField())); Assert.assertEquals("snort", docs.get("snort_2").getDocument().get(getSourceTypeField())); }
@Test public void testCalculateMetaScoresWithDifferentFieldName() { List<Map<String, Object>> alertList = new ArrayList<>(); // add an alert with a threat score alertList.add( Collections.singletonMap(MetaAlertConstants.THREAT_FIELD_DEFAULT, 10.0f)); // create the metaalert Map<String, Object> docMap = new HashMap<>(); docMap.put(MetaAlertConstants.ALERT_FIELD, alertList); Document metaalert = new Document(docMap, "guid", MetaAlertConstants.METAALERT_TYPE, 0L); // Configure a different threat triage score field name AccessConfig accessConfig = new AccessConfig(); accessConfig.setGlobalConfigSupplier(() -> new HashMap<String, Object>() {{ put(Constants.THREAT_SCORE_FIELD_PROPERTY, MetaAlertConstants.THREAT_FIELD_DEFAULT); }}); MetaScores.calculateMetaScores(metaalert, MetaAlertConstants.THREAT_FIELD_DEFAULT, MetaAlertConstants.THREAT_SORT_DEFAULT); assertNotNull(metaalert.getDocument().get(MetaAlertConstants.THREAT_FIELD_DEFAULT)); } }
@Test public void testBuildAddAlertToMetaAlertUpdatesEmpty() { Document metaDoc = new Document( new HashMap<>(), METAALERT_GUID, METAALERT_TYPE, 0L ); metaDoc.getDocument().put( ALERT_FIELD, getRawMaps(buildChildAlerts(1, METAALERT_GUID, null)) ); Map<Document, Optional<String>> actual = dao .buildAddAlertToMetaAlertUpdates(metaDoc, new ArrayList<>()); assertEquals(0, actual.size()); }
private Document createAndIndexDocument(String guid) throws Exception { // create the document Long timestamp = 1526306463050L; Document toCreate = createDocument(guid, timestamp); // index the document Document created = getDao().update(toCreate, Optional.of(SENSOR_NAME)); Assert.assertEquals(toCreate, created); // ensure the document is indexed return findUpdatedDoc(toCreate.getDocument(), guid, SENSOR_NAME); }
@SuppressWarnings("unchecked") @Override public Document createMetaAlert(MetaAlertCreateRequest request) throws InvalidCreateException, IOException { List<GetRequest> alertRequests = request.getAlerts(); if (alertRequests.isEmpty()) { return null; } // Retrieve the documents going into the meta alert and build it Iterable<Document> alerts = indexDao.getAllLatest(alertRequests); Document metaAlert = buildCreateDocument(alerts, request.getGroups(), MetaAlertConstants.ALERT_FIELD); metaAlert.getDocument() .put(getConfig().getSourceTypeField(), MetaAlertConstants.METAALERT_TYPE); return metaAlert; }
@Test public void testUpdate() throws Exception { // create a document to update final String guid = UUID.randomUUID().toString(); final Long timestamp = 1526306463050L; Document toUpdate = createDocument(guid, timestamp); { // update the document and validate Document updated = getDao().update(toUpdate, Optional.of(SENSOR_NAME)); Assert.assertEquals(toUpdate, updated); } { // ensure the document is updated in the index Document indexed = findUpdatedDoc(toUpdate.getDocument(), guid, SENSOR_NAME); Assert.assertEquals(toUpdate, indexed); } }