public void ingest(IndexOperationMessage singleMessage) { this.indexRequests.addAll(singleMessage.getIndexRequests()); this.deIndexRequests.addAll(singleMessage.getDeIndexRequests()); } }
/** * Submit results to index and return the queue messages to be ack'd * */ private List<LegacyQueueMessage> submitToIndex(List<IndexEventResult> indexEventResults, AsyncEventQueueType queueType) { // if nothing came back then return empty list if(indexEventResults==null){ return new ArrayList<>(0); } IndexOperationMessage combined = new IndexOperationMessage(); List<LegacyQueueMessage> queueMessages = indexEventResults.stream() // filter out messages that are not present, they were not processed and put into the results .filter( result -> result.getQueueMessage().isPresent() ) .map(indexEventResult -> { //record the cycle time messageCycle.update(System.currentTimeMillis() - indexEventResult.getCreationTime()); // ingest each index op into our combined, single index op for the index producer if(indexEventResult.getIndexOperationMessage().isPresent()){ combined.ingest(indexEventResult.getIndexOperationMessage().get()); } return indexEventResult.getQueueMessage().get(); }) // collect into a list of QueueMessages that can be ack'd later .collect(Collectors.toList()); queueIndexOperationMessage(combined, queueType); return queueMessages; }
public EntityIndexBatch deindexWithDocId( final String docId ) { String[] indexes = entityIndex.getIndexes(); //get the default index if no alias exists yet if ( indexes == null || indexes.length == 0 ) { throw new IllegalStateException("No indexes exist for " + indexLocationStrategy.getAlias().getWriteAlias()); } if ( logger.isDebugEnabled() ) { logger.debug( "Deindexing to indexes {} with with documentId {} ", indexes, docId ); } container.addDeIndexRequest( new DeIndexOperation( indexes, docId ) ); return this; }
private Observable.Transformer<IndexOperationMessage, IndexOperationMessage> applyCollector(AsyncEventQueueType queueType) { return observable -> observable .collect(() -> new IndexOperationMessage(), (collector, single) -> collector.ingest(single)) .filter(msg -> !msg.isEmpty()) .doOnNext(indexOperation -> { asyncEventService.queueIndexOperationMessage(indexOperation, queueType); }); }
final Set<IndexOperation> indexOperationSet = batch.getIndexRequests(); final Set<DeIndexOperation> deIndexOperationSet = batch.getDeIndexRequests(); indexSizeCounter.dec(deIndexOperationSetSize); final Observable<IndexOperation> index = Observable.from(batch.getIndexRequests()); final Observable<DeIndexOperation> deIndex = Observable.from(batch.getDeIndexRequests()); roundtripTimer.update(System.currentTimeMillis() - processedIndexOp.getCreationTime()); });
null ); assertEquals( 1, indexOperationMessage.getDeIndexRequests().size() );
public EsEntityIndexBatchImpl( final IndexLocationStrategy locationStrategy, final EntityIndex entityIndex ) { this.indexLocationStrategy = locationStrategy; this.entityIndex = entityIndex; this.applicationScope = indexLocationStrategy.getApplicationScope(); this.alias = indexLocationStrategy.getAlias(); //constrained this.container = new IndexOperationMessage(); }
IndexOperationMessage single = new IndexOperationMessage(); && single.isEmpty() ){ logger.warn("No index operation messages came back from event processing for eventType: {}, msgId: {}, msgBody: {}", event.getClass().getSimpleName(), message.getMessageId(), message.getStringBody());
@Test public void testSingleIndexFromSource() { final Entity entity = new Entity( createId( "test" ), UUIDGenerator.newTimeUUID() ); entity.setField( new StringField( "string", "foo" ) ); final Edge collectionEdge = createCollectionEdge( applicationScope.getApplication(), "tests", entity.getId() ); //write the edge graphManager.writeEdge( collectionEdge ).toBlocking().last(); //index the edge final Observable<IndexOperationMessage> indexed = indexService.indexEntity( applicationScope, entity ); //real users should never call to blocking, we're not sure what we'll get final IndexOperationMessage results = indexed.toBlocking().last(); indexProducer.put(results).subscribe(); final Set<IndexOperation> indexRequests = results.getIndexRequests(); //ensure our value made it to the index request final IndexOperation indexRequest = indexRequests.iterator().next(); assertNotNull( indexRequest ); }
/** * Queue up an indexOperationMessage for multi region execution * @param indexOperationMessage * @param queueType */ public void queueIndexOperationMessage(final IndexOperationMessage indexOperationMessage, AsyncEventQueueType queueType) { // don't try to produce something with nothing if(indexOperationMessage == null || indexOperationMessage.isEmpty()){ return; } final String jsonValue = ObjectJsonSerializer.INSTANCE.toString( indexOperationMessage ); final UUID newMessageId = UUIDGenerator.newTimeUUID(); final int expirationTimeInSeconds = ( int ) TimeUnit.MILLISECONDS.toSeconds( indexProcessorFig.getIndexMessageTtl() ); //write to the map in ES esMapPersistence.putString( newMessageId.toString(), jsonValue, expirationTimeInSeconds ); //now queue up the index message final ElasticsearchIndexEvent elasticsearchIndexEvent = new ElasticsearchIndexEvent(queueFig.getPrimaryRegion(), newMessageId ); //send to the topic so all regions index the batch if (logger.isTraceEnabled()) { logger.trace("Offering ElasticsearchIndexEvent for message {}", newMessageId); } offerTopic( elasticsearchIndexEvent, queueType ); }
@Override public EntityIndexBatch index( final IndexEdge indexEdge, final Entity entity, final Optional<Set<String>> fieldsToIndex ) { IndexValidationUtils.validateIndexEdge(indexEdge); ValidationUtils.verifyEntityWrite(entity); ValidationUtils.verifyVersion( entity.getVersion() ); final String writeAlias = alias.getWriteAlias(); if ( logger.isDebugEnabled() ) { logger.debug( "Indexing to alias {} with scope {} on edge {} with entity data {}", writeAlias, applicationScope, indexEdge, entity.getFieldMap().keySet() ); } //add app id for indexing container.addIndexRequest(new IndexOperation(writeAlias, applicationScope, indexEdge, entity,fieldsToIndex)); return this; }
private Observable.Transformer<IndexOperationMessage, IndexOperationMessage> applyCollector(AsyncEventQueueType queueType) { return observable -> observable .collect(() -> new IndexOperationMessage(), (collector, single) -> collector.ingest(single)) .filter(msg -> !msg.isEmpty()) .doOnNext(indexOperation -> { asyncEventService.queueIndexOperationMessage(indexOperation, queueType); }); }
assertEquals( 1, indexOperationMessage.getDeIndexRequests().size() );
private IndexOperationMessage handleEdgeIndex(final LegacyQueueMessage message) { Preconditions.checkNotNull( message, "Queue Message cannot be null for handleEdgeIndex" ); final AsyncEvent event = (AsyncEvent) message.getBody(); Preconditions.checkNotNull( message, "QueueMessage Body cannot be null for handleEdgeIndex" ); Preconditions.checkArgument(event instanceof EdgeIndexEvent, String.format("Event Type for handleEdgeIndex must be EDGE_INDEX, got %s", event.getClass())); final EdgeIndexEvent edgeIndexEvent = ( EdgeIndexEvent ) event; final EntityCollectionManager ecm = entityCollectionManagerFactory.createCollectionManager( edgeIndexEvent.getApplicationScope() ); // default this observable's return to empty index operation message if nothing is emitted return ecm.load( edgeIndexEvent.getEntityId() ) .flatMap( loadedEntity -> eventBuilder.buildNewEdge(edgeIndexEvent.getApplicationScope(), loadedEntity, edgeIndexEvent.getEdge())) .toBlocking().lastOrDefault(new IndexOperationMessage()); }
if (indexOperationMessage == null || indexOperationMessage.isEmpty()){ throw new RuntimeException( "IndexOperationMessage cannot be null or empty after retrieving from map persistence");
@Override public int size() { return container.getDeIndexRequests().size() + container.getIndexRequests().size(); } }
protected List<LegacyQueueMessage> submitToIndex(List<IndexEventResult> indexEventResults, boolean forUtilityQueue) { // if nothing came back then return empty list if(indexEventResults==null){ return new ArrayList<>(0); } IndexOperationMessage combined = new IndexOperationMessage(); List<LegacyQueueMessage> queueMessages = indexEventResults.stream() // filter out messages that are not present, they were not processed and put into the results .filter( result -> result.getQueueMessage().isPresent() ) .map(indexEventResult -> { //record the cycle time getMessageCycle().update(System.currentTimeMillis() - indexEventResult.getCreationTime()); // ingest each index op into our combined, single index op for the index producer if(indexEventResult.getIndexOperationMessage().isPresent()){ combined.ingest(indexEventResult.getIndexOperationMessage().get()); } return indexEventResult.getQueueMessage().get(); }) // collect into a list of QueueMessages that can be ack'd later .collect(Collectors.toList()); // dispatch to ES ElasticsearchIndexEvent elasticsearchIndexEvent = getESIndexEvent(combined); handleIndexOperation(elasticsearchIndexEvent); return queueMessages; }
private IndexOperationMessage handleEntityIndexUpdate(final LegacyQueueMessage message) { Preconditions.checkNotNull( message, "Queue Message cannot be null for handleEntityIndexUpdate" ); final AsyncEvent event = ( AsyncEvent ) message.getBody(); Preconditions.checkNotNull(message, "QueueMessage Body cannot be null for handleEntityIndexUpdate"); Preconditions.checkArgument(event instanceof EntityIndexEvent, String.format("Event Type for handleEntityIndexUpdate must be ENTITY_INDEX, got %s", event.getClass())); final EntityIndexEvent entityIndexEvent = (EntityIndexEvent) event; //process the entity immediately //only process the same version, otherwise ignore final EntityIdScope entityIdScope = entityIndexEvent.getEntityIdScope(); final ApplicationScope applicationScope = entityIdScope.getApplicationScope(); final Id entityId = entityIdScope.getId(); final long updatedAfter = entityIndexEvent.getUpdatedAfter(); final EntityIndexOperation entityIndexOperation = new EntityIndexOperation( applicationScope, entityId, updatedAfter); // default this observable's return to empty index operation message if nothing is emitted return eventBuilder.buildEntityIndex( entityIndexOperation ) .toBlocking().lastOrDefault(new IndexOperationMessage()); }
@Override public EntityIndexBatch deindex( final SearchEdge searchEdge, final Id id, final UUID version ) { IndexValidationUtils.validateSearchEdge(searchEdge); ValidationUtils.verifyIdentity(id); ValidationUtils.verifyVersion( version ); String[] indexes = entityIndex.getIndexes(); //get the default index if no alias exists yet if ( indexes == null || indexes.length == 0 ) { throw new IllegalStateException("No indexes exist for " + indexLocationStrategy.getAlias().getWriteAlias()); } if ( logger.isDebugEnabled() ) { logger.debug( "Deindexing to indexes {} with scope {} on edge {} with id {} and version {} ", indexes, applicationScope, searchEdge, id, version ); } container.addDeIndexRequest(new DeIndexOperation(indexes, applicationScope, searchEdge, id, version)); return this; }
public Observable<IndexOperationMessage> put( IndexOperationMessage message ) { Preconditions.checkNotNull(message, "Message cannot be null"); indexSizeCounter.inc(message.getDeIndexRequests().size()); indexSizeCounter.inc(message.getIndexRequests().size()); return processBatch(message); }