public EsEntityIndexBatchImpl( final IndexLocationStrategy locationStrategy, final EntityIndex entityIndex ) { this.indexLocationStrategy = locationStrategy; this.entityIndex = entityIndex; this.applicationScope = indexLocationStrategy.getApplicationScope(); this.alias = indexLocationStrategy.getAlias(); //constrained this.container = new IndexOperationMessage(); }
/** * Submit results to index and return the queue messages to be ack'd * */ private List<LegacyQueueMessage> submitToIndex(List<IndexEventResult> indexEventResults, AsyncEventQueueType queueType) { // if nothing came back then return empty list if(indexEventResults==null){ return new ArrayList<>(0); } IndexOperationMessage combined = new IndexOperationMessage(); List<LegacyQueueMessage> queueMessages = indexEventResults.stream() // filter out messages that are not present, they were not processed and put into the results .filter( result -> result.getQueueMessage().isPresent() ) .map(indexEventResult -> { //record the cycle time messageCycle.update(System.currentTimeMillis() - indexEventResult.getCreationTime()); // ingest each index op into our combined, single index op for the index producer if(indexEventResult.getIndexOperationMessage().isPresent()){ combined.ingest(indexEventResult.getIndexOperationMessage().get()); } return indexEventResult.getQueueMessage().get(); }) // collect into a list of QueueMessages that can be ack'd later .collect(Collectors.toList()); queueIndexOperationMessage(combined, queueType); return queueMessages; }
private IndexOperationMessage handleEdgeIndex(final LegacyQueueMessage message) { Preconditions.checkNotNull( message, "Queue Message cannot be null for handleEdgeIndex" ); final AsyncEvent event = (AsyncEvent) message.getBody(); Preconditions.checkNotNull( message, "QueueMessage Body cannot be null for handleEdgeIndex" ); Preconditions.checkArgument(event instanceof EdgeIndexEvent, String.format("Event Type for handleEdgeIndex must be EDGE_INDEX, got %s", event.getClass())); final EdgeIndexEvent edgeIndexEvent = ( EdgeIndexEvent ) event; final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager( edgeIndexEvent.getApplicationScope() ); // default this observable's return to empty index operation message if nothing is emitted return ecm.load( edgeIndexEvent.getEntityId() ) .flatMap( loadedEntity -> eventBuilder.buildNewEdge(edgeIndexEvent.getApplicationScope(), loadedEntity, edgeIndexEvent.getEdge())) .toBlocking().lastOrDefault(new IndexOperationMessage()); }
protected List<LegacyQueueMessage> submitToIndex(List<IndexEventResult> indexEventResults, boolean forUtilityQueue) { // if nothing came back then return empty list if(indexEventResults==null){ return new ArrayList<>(0); } IndexOperationMessage combined = new IndexOperationMessage(); List<LegacyQueueMessage> queueMessages = indexEventResults.stream() // filter out messages that are not present, they were not processed and put into the results .filter( result -> result.getQueueMessage().isPresent() ) .map(indexEventResult -> { //record the cycle time getMessageCycle().update(System.currentTimeMillis() - indexEventResult.getCreationTime()); // ingest each index op into our combined, single index op for the index producer if(indexEventResult.getIndexOperationMessage().isPresent()){ combined.ingest(indexEventResult.getIndexOperationMessage().get()); } return indexEventResult.getQueueMessage().get(); }) // collect into a list of QueueMessages that can be ack'd later .collect(Collectors.toList()); // dispatch to ES ElasticsearchIndexEvent elasticsearchIndexEvent = getESIndexEvent(combined); handleIndexOperation(elasticsearchIndexEvent); return queueMessages; }
private IndexOperationMessage handleEntityIndexUpdate(final LegacyQueueMessage message) { Preconditions.checkNotNull( message, "Queue Message cannot be null for handleEntityIndexUpdate" ); final AsyncEvent event = ( AsyncEvent ) message.getBody(); Preconditions.checkNotNull(message, "QueueMessage Body cannot be null for handleEntityIndexUpdate"); Preconditions.checkArgument(event instanceof EntityIndexEvent, String.format("Event Type for handleEntityIndexUpdate must be ENTITY_INDEX, got %s", event.getClass())); final EntityIndexEvent entityIndexEvent = (EntityIndexEvent) event; //process the entity immediately //only process the same version, otherwise ignore final EntityIdScope entityIdScope = entityIndexEvent.getEntityIdScope(); final ApplicationScope applicationScope = entityIdScope.getApplicationScope(); final Id entityId = entityIdScope.getId(); final long updatedAfter = entityIndexEvent.getUpdatedAfter(); final EntityIndexOperation entityIndexOperation = new EntityIndexOperation( applicationScope, entityId, updatedAfter); // default this observable's return to empty index operation message if nothing is emitted return eventBuilder.buildEntityIndex( entityIndexOperation ) .toBlocking().lastOrDefault(new IndexOperationMessage()); }
private Observable.Transformer<IndexOperationMessage, IndexOperationMessage> applyCollector(AsyncEventQueueType queueType) { return observable -> observable .collect(() -> new IndexOperationMessage(), (collector, single) -> collector.ingest(single)) .filter(msg -> !msg.isEmpty()) .doOnNext(indexOperation -> { asyncEventService.queueIndexOperationMessage(indexOperation, queueType); }); }
private Observable.Transformer<IndexOperationMessage, IndexOperationMessage> applyCollector(AsyncEventQueueType queueType) { return observable -> observable .collect(() -> new IndexOperationMessage(), (collector, single) -> collector.ingest(single)) .filter(msg -> !msg.isEmpty()) .doOnNext(indexOperation -> { asyncEventService.queueIndexOperationMessage(indexOperation, queueType); }); }
public IndexOperationMessage handleDeIndexOldVersionEvent ( final DeIndexOldVersionsEvent deIndexOldVersionsEvent){ final ApplicationScope applicationScope = deIndexOldVersionsEvent.getEntityIdScope().getApplicationScope(); final Id entityId = deIndexOldVersionsEvent.getEntityIdScope().getId(); final UUID markedVersion = deIndexOldVersionsEvent.getMarkedVersion(); // default this observable's return to empty index operation message if nothing is emitted return eventBuilder.deIndexOldVersions( applicationScope, entityId, markedVersion ) .toBlocking().lastOrDefault(new IndexOperationMessage()); }
@Override public IndexOperationMessage buildDeleteEdge( final ApplicationScope applicationScope, final Edge edge ) { if (logger.isDebugEnabled()) { logger.debug("Deleting in app scope {} with edge {}", applicationScope, edge); } final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope ); final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager( applicationScope ); final IndexOperationMessage combined = new IndexOperationMessage(); gm.deleteEdge( edge ) .doOnNext( deletedEdge -> { logger.debug("Processing deleted edge for de-indexing {}", deletedEdge); // get ALL versions of the target node as any connection from this source node needs to be removed ecm.getVersionsFromMaxToMin(deletedEdge.getTargetNode(), UUIDUtils.newTimeUUID()) .doOnNext(mvccLogEntry -> { if (logger.isDebugEnabled()) { logger.debug("Adding edge {} mvccLogEntry {} to de-index batch", deletedEdge.getTargetNode(), mvccLogEntry); } combined.ingest( indexService .deIndexEdge(applicationScope, deletedEdge, mvccLogEntry.getEntityId(), mvccLogEntry.getVersion()) .toBlocking().lastOrDefault(new IndexOperationMessage())); }).toBlocking().lastOrDefault(null); }).toBlocking().lastOrDefault(null); return combined; }
IndexOperationMessage single = new IndexOperationMessage();
return new IndexOperationMessage(); return new IndexOperationMessage(); IndexOperationMessage combined = new IndexOperationMessage(); indexService .deIndexEdge(applicationScope, markedEdge, mvccLogEntry.getEntityId(), mvccLogEntry.getVersion()) .toBlocking().lastOrDefault(new IndexOperationMessage())); indexService .deIndexEdge(applicationScope, markedEdge, logEntry.getEntityId(), logEntry.getVersion()) .toBlocking().lastOrDefault(new IndexOperationMessage())); });