protected String getIndexName(Document update, String indexPostFix) throws IOException { return findIndexNameByGUID(update.getGuid(), update.getSensorType()) .orElse(ElasticsearchUtils.getIndexName(update.getSensorType(), indexPostFix, null)); }
protected Entry<Document, Optional<String>> findMetaEntry( Map<Document, Optional<String>> expected) { for (Entry<Document, Optional<String>> entry : expected.entrySet()) { if (entry.getKey().getSensorType().equals(METAALERT_TYPE)) { return entry; } } return null; }
SolrInputDocument solrInputDocument = SolrUtilities.toSolrInputDocument(entry.getKey()); Optional<String> index = SolrUtilities .getIndex(config.getIndexSupplier(), entry.getKey().getSensorType(), entry.getValue()); if (index.isPresent()) { Collection<SolrInputDocument> solrInputDocuments = solrCollectionUpdates collectionsUpdated.add(index.get()); } else { String lookupIndex = config.getIndexSupplier().apply(entry.getKey().getSensorType()); Collection<SolrInputDocument> solrInputDocuments = solrCollectionUpdates .getOrDefault(lookupIndex, new ArrayList<>());
public static Document fromJSON(Map<String, Object> json) { String guid = getGUID(json); Long timestamp = getTimestamp(json).orElse(0L); String sensorType = getSensorType(json); return new Document(json, guid, sensorType, timestamp); }
protected Map<Document, Optional<String>> removeMetaEntry( Map<Document, Optional<String>> updates) { Map<Document, Optional<String>> filteredUpdates = new HashMap<>(); for (Entry<Document, Optional<String>> entry : updates.entrySet()) { if (!(entry.getKey().getSensorType().equals(METAALERT_TYPE))) { filteredUpdates.put(entry.getKey(), entry.getValue()); } } return filteredUpdates; }
/** * Copy constructor * @param other The document to be copied. */ public Document(Document other) { this(new HashMap<>(other.getDocument()), other.getGuid(), other.getSensorType(), other.getTimestamp(), other.getDocumentID().orElse(null)); }
private DocumentContainer removeCommentFromAlert(IndexDao indexDao, CommentAddRemoveRequest request, Document latest) { DocumentContainer container; try { Document document = indexDao.removeCommentFromAlert(request, latest); container = new DocumentContainer(document); LOG.debug("Removed comment from alert; indexDao={}, guid={}, sensorType={}, document={}", ClassUtils.getShortClassName(indexDao.getClass()), document.getGuid(), document.getSensorType(), document); } catch (Throwable e) { container = new DocumentContainer(e); LOG.error("Unable to remove comment from alert; indexDao={}, error={}", ClassUtils.getShortClassName(indexDao.getClass()), ExceptionUtils.getRootCauseMessage(e)); } return container; }
private DocumentContainer addCommentToAlert(IndexDao indexDao, CommentAddRemoveRequest request, Document latest) { DocumentContainer container; try { Document document = indexDao.addCommentToAlert(request, latest); container = new DocumentContainer(document); LOG.debug("Added comment to alert; indexDao={}, guid={}, sensorType={}, document={}", ClassUtils.getShortClassName(indexDao.getClass()), document.getGuid(), document.getSensorType(), document); } catch (Throwable e) { container = new DocumentContainer(e); LOG.error("Unable to add comment to alert; indexDao={}, error={}", ClassUtils.getShortClassName(indexDao.getClass()), ExceptionUtils.getRootCauseMessage(e)); } return container; }
@Override public Document update(Document update, Optional<String> index) throws IOException { if (MetaAlertConstants.METAALERT_TYPE.equals(update.getSensorType())) {
private IndexRequest createRequest(D document, String index) { if(document.getTimestamp() == null) { throw new IllegalArgumentException("Document must contain the timestamp"); } // if updating an existing document, the doc ID should be defined. // if creating a new document, set the doc ID to null to allow Elasticsearch to generate one. String docId = document.getDocumentID().orElse(null); if(LOG.isDebugEnabled() && document.getDocumentID().isPresent()) { LOG.debug("Updating existing document with known doc ID; docID={}, guid={}, sensorType={}", docId, document.getGuid(), document.getSensorType()); } else if(LOG.isDebugEnabled()) { LOG.debug("Creating a new document, doc ID not yet known; guid={}, sensorType={}", document.getGuid(), document.getSensorType()); } return new IndexRequest() .source(document.getDocument()) .type(document.getSensorType() + "_doc") .index(index) .id(docId) .index(index) .timestamp(document.getTimestamp().toString()); }
@Override public Document update(Document update, Optional<String> rawIndex) throws IOException { Document newVersion = update; // Handle any case where we're given comments in Map form, instead of raw String Object commentsObj = update.getDocument().get(COMMENTS_FIELD); if ( commentsObj instanceof List && ((List<Object>) commentsObj).size() > 0 && ((List<Object>) commentsObj).get(0) instanceof Map) { newVersion = new Document(update); convertCommentsToRaw(newVersion.getDocument()); } try { SolrInputDocument solrInputDocument = SolrUtilities.toSolrInputDocument(newVersion); Optional<String> index = SolrUtilities .getIndex(config.getIndexSupplier(), newVersion.getSensorType(), rawIndex); if (index.isPresent()) { this.client.add(index.get(), solrInputDocument); this.client.commit(index.get()); } else { throw new IllegalStateException("Index must be specified or inferred."); } } catch (SolrServerException e) { throw new IOException(e); } return newVersion; }
protected Put buildPut(Document update) throws IOException { Key k = new Key(update.getGuid(), update.getSensorType()); Put put = new Put(Key.toBytes(k)); long ts = update.getTimestamp() == null || update.getTimestamp() == 0 ? System.currentTimeMillis() : update.getTimestamp(); byte[] columnQualifier = Bytes.toBytes(ts); byte[] doc = JSONUtils.INSTANCE.toJSONPretty(update.getDocument()); put.addColumn(cf, columnQualifier, doc); return put; }
@Override public Document update(Document update, Optional<String> index) throws IOException { for (Map.Entry<String, List<String>> kv : BACKING_STORE.entrySet()) { if (kv.getKey().startsWith(update.getSensorType())) { for (Iterator<String> it = kv.getValue().iterator(); it.hasNext(); ) { String doc = it.next(); Map<String, Object> docParsed = parse(doc); if (docParsed.getOrDefault(Constants.GUID, "").equals(update.getGuid())) { it.remove(); } } kv.getValue().add(JSONUtils.INSTANCE.toJSON(update.getDocument(), true)); } } return update; }
if (MetaAlertConstants.METAALERT_TYPE.equals(update.getSensorType())) {
return false; if (!expected.getSensorType().equals(actual.getSensorType())) { return false;
@Test public void testRemoveComments() throws Exception { String guid = "testRemoveComments"; createAndIndexDocument(guid); // add a comment on the document Document withComments = addAlertComment(guid, "comment", "user1", 1526401584951L); Assert.assertEquals(1, getComments(withComments).size()); // ensure the comment was added to the document in the index Document indexedWithComments = findUpdatedDoc(withComments.getDocument(), withComments.getGuid(), withComments.getSensorType()); Assert.assertEquals(1, getComments(indexedWithComments).size()); // remove a comment from the document AlertComment toRemove = getComments(withComments).get(0); Document noComments = removeAlertComment(guid, toRemove.getComment(), toRemove.getUsername(), toRemove.getTimestamp()); Assert.assertEquals(0, getComments(noComments).size()); // ensure the comment was removed from the index Document indexedNoComments = findUpdatedDoc(noComments.getDocument(), withComments.getGuid(), withComments.getSensorType()); Assert.assertEquals(0, getComments(indexedNoComments).size()); }
@Test public void testWrite() throws Exception { // create some documents to write List<Document> documents = new ArrayList<>(); for(int i=0; i<10; i++) { Document document = Document.fromJSON(createMessage()); documents.add(document); } // write the documents for(Document doc: documents) { writer.addDocument(doc, "bro_index"); } writer.write(); // ensure the documents were written for(Document expected: documents) { Document actual = retrieveDao.getLatest(expected.getGuid(), expected.getSensorType()); assertNotNull("No document found", actual); assertEquals(expected.getGuid(), actual.getGuid()); assertEquals(expected.getSensorType(), actual.getSensorType()); assertEquals(expected.getDocument(), actual.getDocument()); assertTrue(actual.getDocumentID().isPresent()); // the document ID and GUID should not be the same, since the document ID was auto-generated assertNotEquals(actual.getDocument(), actual.getGuid()); } }