@Override public EntityIndex getEntityIndex( ApplicationScope applicationScope) { IndexLocationStrategy indexLocationStrategy = indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope); return eif.createEntityIndex( indexLocationStrategy ); }
private void handleInitializeApplicationIndex(final AsyncEvent event, final LegacyQueueMessage message) { Preconditions.checkNotNull(message, "Queue Message cannot be null for handleInitializeApplicationIndex"); Preconditions.checkArgument(event instanceof InitializeApplicationIndexEvent, String.format("Event Type for handleInitializeApplicationIndex must be APPLICATION_INDEX, got %s", event.getClass())); final InitializeApplicationIndexEvent initializeApplicationIndexEvent = ( InitializeApplicationIndexEvent ) event; final IndexLocationStrategy indexLocationStrategy = initializeApplicationIndexEvent.getIndexLocationStrategy(); final EntityIndex index = entityIndexFactory.createEntityIndex( indexLocationStrategy ); index.initialize(); }
@Override public long getSize(ApplicationScope applicationScope, SearchEdge edge) { final IndexLocationStrategy indexLocationStrategy = indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope); EntityIndex entityIndex = entityIndexFactory.createEntityIndex(indexLocationStrategy); return entityIndex.getTotalEntitySizeInBytes(edge); }
@Override public Observable<IndexOperationMessage> deIndexEdge(final ApplicationScope applicationScope, final Edge edge, final Id entityId, final UUID entityVersion){ if (logger.isTraceEnabled()) { logger.trace("deIndexEdge edge={} entityId={} entityVersion={}", edge.toString(), entityId.toString(), entityVersion.toString()); } final EntityIndex ei = entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope)); final EntityIndexBatch entityBatch = ei.createBatch(); entityBatch.deindex(generateScopeFromSource( edge ), entityId, entityVersion); return Observable.just(entityBatch.build()); }
/** * this method will call initialize for each message, since we are caching the entity indexes, * we don't worry about aggregating by app id * @param indexOperationMessage */ private void initializeEntityIndexes(final IndexOperationMessage indexOperationMessage) { // create a set so we can have a unique list of appIds for which we call createEntityIndex Set<UUID> appIds = new HashSet<>(); // loop through all indexRequests and add the appIds to the set indexOperationMessage.getIndexRequests().forEach(req -> { UUID appId = IndexingUtils.getApplicationIdFromIndexDocId(req.documentId); appIds.add(appId); }); // loop through all deindexRequests and add the appIds to the set indexOperationMessage.getDeIndexRequests().forEach(req -> { UUID appId = IndexingUtils.getApplicationIdFromIndexDocId(req.documentId); appIds.add(appId); }); // for each of the appIds in the unique set, call create entity index to ensure the aliases are created appIds.forEach(appId -> { ApplicationScope appScope = CpNamingUtils.getApplicationScope(appId); entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(appScope)); } ); }
/** * Generate an observable for our appliation scope */ private Observable<ApplicationScope> getApplications( final Optional<EdgeScope> cursor, final Optional<ApplicationScope> appId ) { //cursor is present use it and skip until we hit that app if (cursor.isPresent()) { final EdgeScope cursorValue = cursor.get(); //we have a cursor and an application scope that was used. return allApplicationsObservable.getData().skipWhile( applicationScope -> !cursorValue.getApplicationScope().equals(applicationScope)); } //this is intentional. If else if (appId.isPresent()) { return Observable.just(appId.get()) .doOnNext(appScope -> { //make sure index is initialized on rebuild entityIndexFactory.createEntityIndex( indexLocationStrategyFactory.getIndexLocationStrategy(appScope) ).initialize(); }); } return allApplicationsObservable.getData() .doOnNext(appScope -> { //make sure index is initialized on rebuild entityIndexFactory.createEntityIndex( indexLocationStrategyFactory.getIndexLocationStrategy(appScope) ).initialize(); }); }
@Override public Observable<IndexOperationMessage> indexEdge( final ApplicationScope applicationScope, final Entity entity, final Edge edge ) { final Observable<IndexOperationMessage> batches = Observable.just( edge ).map( observableEdge -> { //if the node is the target node, generate our scope correctly if ( edge.getTargetNode().equals( entity.getId() ) ) { return generateScopeFromSource( edge ); } throw new IllegalArgumentException("target not equal to entity + "+entity.getId()); } ).map( indexEdge -> { final EntityIndex ei = entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) ); final EntityIndexBatch batch = ei.createBatch(); if (logger.isDebugEnabled()) { logger.debug("adding edge {} to batch for entity {}", indexEdge, entity); } Optional<Set<String>> fieldsToIndex = getFilteredStringObjectMap( indexEdge ); batch.index( indexEdge, entity ,fieldsToIndex); return batch.build(); } ); return ObservableTimer.time( batches, addTimer ); }
@Override public long getApplicationSize(ApplicationScope applicationScope) { final IndexLocationStrategy indexLocationStrategy = indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope); EntityIndex entityIndex = entityIndexFactory.createEntityIndex(indexLocationStrategy); GraphManager graphManager = graphManagerFactory.createEdgeManager(applicationScope); Long sum = ObservableTimer.time( MathObservable.sumLong( graphManager.getEdgeTypesFromSource(new SimpleSearchEdgeType(applicationScope.getApplication(), CpNamingUtils.EDGE_COLL_PREFIX, Optional.<String>absent())) .map(type -> CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(), type)) .map(edge -> entityIndex.getTotalEntitySizeInBytes(edge)) ), sumTimer).toBlocking().last(); return sum.longValue(); }
@Override public Observable<FilterResult<Id>> call( final Observable<FilterResult<Candidate>> filterResultObservable ) { /** * A bit kludgy from old 1.0 -> 2.0 apis. Refactor this as we clean up our lower levels and create new results * objects */ final ApplicationScope applicationScope = pipelineContext.getApplicationScope(); final EntityCollectionManager entityCollectionManager = entityCollectionManagerFactory.createCollectionManager( applicationScope ); final EntityIndex applicationIndex = entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope)); final Observable<FilterResult<Id>> searchIdSetObservable = filterResultObservable.buffer( pipelineContext.getLimit() ).flatMap( candidateResults -> { //flatten toa list of ids to load final Observable<List<Id>> candidateIds = Observable.from( candidateResults ).map( candidate -> candidate.getValue().getCandidateResult().getId() ).toList(); //load the ids final Observable<VersionSet> versionSetObservable = candidateIds.flatMap( ids -> entityCollectionManager.getLatestVersion( ids ) ); //now we have a collection, validate our canidate set is correct. return versionSetObservable.map( entitySet -> new EntityCollector( applicationIndex.createBatch(), entitySet, candidateResults, indexProducer ) ).doOnNext( entityCollector -> entityCollector.merge() ).flatMap( entityCollector -> Observable.from( entityCollector.collectResults() ) ); } ); return searchIdSetObservable; }
@Override public Map<String, Long> getEachCollectionSize(ApplicationScope applicationScope) { final IndexLocationStrategy indexLocationStrategy = indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope); EntityIndex entityIndex = entityIndexFactory.createEntityIndex(indexLocationStrategy); GraphManager graphManager = graphManagerFactory.createEdgeManager(applicationScope); Map<String,Long> sumMap = ObservableTimer.time( graphManager.getEdgeTypesFromSource(new SimpleSearchEdgeType(applicationScope.getApplication(), CpNamingUtils.EDGE_COLL_PREFIX, Optional.<String>absent())) .collect(() -> new HashMap<String,Long>(), ((map, type) -> { SearchEdge edge = CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(), type); final String collectionName = CpNamingUtils.getCollectionNameFromEdgeName(type); long sumType = entityIndex.getTotalEntitySizeInBytes(edge); map.put(collectionName,sumType); }) ) , sumTimer).toBlocking().last(); return sumMap; }
@Override public Observable<IndexOperationMessage> deIndexOldVersions(final ApplicationScope applicationScope, final Id entityId, final List<UUID> versions) { final EntityIndex ei = entityIndexFactory. createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) ); // use LONG.MAX_VALUE in search edge because this value is not used elsewhere in lower code for de-indexing // previously .timsetamp() was used on entityId, but some entities do not have type-1 UUIDS ( legacy data) final SearchEdge searchEdgeFromSource = createSearchEdgeFromSource( new SimpleEdge( applicationScope.getApplication(), CpNamingUtils.getEdgeTypeFromCollectionName( InflectionUtils.pluralize( entityId.getType() ) ), entityId, Long.MAX_VALUE ) ); final EntityIndexBatch batch = ei.createBatch(); versions.forEach( version -> { batch.deindex(searchEdgeFromSource, entityId, version); }); return Observable.just(batch.build()); }
@Before public void before(){ appId = new SimpleId(UUID.randomUUID(), "application" ); IndexLocationStrategy strategy = new TestIndexIdentifier(cassandraFig,fig,new ApplicationScopeImpl(appId)); entityIndex = eif.createEntityIndex( strategy ); }
@Before public void setup(){ appId = new SimpleId(UUID.randomUUID(), "application" ); IndexLocationStrategy strategy = new TestIndexIdentifier(cassandraFig,fig,new ApplicationScopeImpl(appId)); entityIndex = eif.createEntityIndex( strategy ); }
@Before public void setup(){ appId = new SimpleId(UUID.randomUUID(), "application" ); IndexLocationStrategy strategy =new TestIndexIdentifier(cassandraFig,fig,new ApplicationScopeImpl(appId)); entityIndex = eif.createEntityIndex( strategy ); }
/** * Delete app index */ private void deleteIndex(UUID appUuid) { Injector injector = SpringResource.getInstance().getBean(Injector.class); IndexLocationStrategyFactory indexLocationStrategyFactory = injector.getInstance(IndexLocationStrategyFactory.class); EntityIndexFactory eif = injector.getInstance(EntityIndexFactory.class); Id appId = new SimpleId(appUuid, Schema.TYPE_APPLICATION); ApplicationScope scope = new ApplicationScopeImpl(appId); EntityIndex ei = eif.createEntityIndex( indexLocationStrategyFactory.getIndexLocationStrategy(scope) ); ei.deleteApplication().toBlocking().lastOrDefault(null); app.waitForQueueDrainAndRefreshIndex(); }
@Override public Observable<IndexOperationMessage> deleteIndexEdge( final ApplicationScope applicationScope, final Edge edge ) { final Observable<IndexOperationMessage> batches = Observable.just( edge ).map( edgeValue -> { final EntityIndex ei = entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) ); EntityIndexBatch batch = ei.createBatch(); //review why generating the Scope from the Source and the target node makes sense. final IndexEdge fromSource = generateScopeFromSource( edge ); final Id targetId = edge.getTargetNode(); CandidateResults targetEdgesToBeDeindexed = ei.getAllEdgeDocuments( fromSource, targetId ); //1. Feed the observable the candidate results you got back. Since it now does the aggregation for you // you don't need to worry about putting your code in a do while. batch = deindexBatchIteratorResolver( fromSource, targetEdgesToBeDeindexed, batch ); final IndexEdge fromTarget = generateScopeFromTarget( edge ); final Id sourceId = edge.getSourceNode(); CandidateResults sourceEdgesToBeDeindexed = ei.getAllEdgeDocuments( fromTarget, sourceId ); batch = deindexBatchIteratorResolver( fromTarget, sourceEdgesToBeDeindexed, batch ); return batch.build(); } ); return ObservableTimer.time( batches, addTimer ); }
/** * Create an application with the given app name and org name * @param orgName * @param appName */ public void createApplication(final String orgName, final String appName) throws Exception { this.orgName = orgName; this.appName = appName; id = setup.createApplication( orgName, appName ); managementEm = setup.getEmf().getEntityManager(setup.getEmf().getManagementAppId()); assertNotNull(id); em = setup.getEmf().getEntityManager(id); Injector injector = setup.getInjector(); IndexLocationStrategyFactory indexLocationStrategyFactory = injector.getInstance(IndexLocationStrategyFactory.class); entityIndexFactory = injector.getInstance(EntityIndexFactory.class); applicationIndex = entityIndexFactory.createEntityIndex( indexLocationStrategyFactory.getIndexLocationStrategy(CpNamingUtils.getApplicationScope(id)) ); assertNotNull(em); logger.info( "Created new application {} in organization {}", appName, orgName ); //wait for the index before proceeding waitForQueueDrainAndRefreshIndex(500); }
/** * Go around EntityManager and execute query directly against Core Persistence. * Results may include stale index entries. */ private CandidateResults queryCollectionCp( final String collName, final String type, final String query ) { EntityManager em = app.getEntityManager(); EntityIndexFactory eif = SpringResource.getInstance().getBean( Injector.class ).getInstance( EntityIndexFactory.class ); ApplicationScope as = new ApplicationScopeImpl( new SimpleId( em.getApplicationId(), TYPE_APPLICATION ) ); IndexLocationStrategyFactory indexLocationStrategyFactory = SpringResource.getInstance().getBean( Injector.class ).getInstance(IndexLocationStrategyFactory.class); EntityIndex ei = eif.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(as)); final Id rootId = createId(em.getApplicationId(), TYPE_APPLICATION); SearchEdge is = CpNamingUtils.createCollectionSearchEdge( rootId, collName ); return ei.search( is, SearchTypes.fromTypes( type ), query, 1000, 0, false ); }
entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) );
entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) );