@Override public Observable<IndexOperationMessage> deIndexEdge(final ApplicationScope applicationScope, final Edge edge, final Id entityId, final UUID entityVersion){ if (logger.isTraceEnabled()) { logger.trace("deIndexEdge edge={} entityId={} entityVersion={}", edge.toString(), entityId.toString(), entityVersion.toString()); } final EntityIndex ei = entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope)); final EntityIndexBatch entityBatch = ei.createBatch(); entityBatch.deindex(generateScopeFromSource( edge ), entityId, entityVersion); return Observable.just(entityBatch.build()); }
@Override public Observable<IndexOperationMessage> indexEdge( final ApplicationScope applicationScope, final Entity entity, final Edge edge ) { final Observable<IndexOperationMessage> batches = Observable.just( edge ).map( observableEdge -> { //if the node is the target node, generate our scope correctly if ( edge.getTargetNode().equals( entity.getId() ) ) { return generateScopeFromSource( edge ); } throw new IllegalArgumentException("target not equal to entity + "+entity.getId()); } ).map( indexEdge -> { final EntityIndex ei = entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) ); final EntityIndexBatch batch = ei.createBatch(); if (logger.isDebugEnabled()) { logger.debug("adding edge {} to batch for entity {}", indexEdge, entity); } Optional<Set<String>> fieldsToIndex = getFilteredStringObjectMap( indexEdge ); batch.index( indexEdge, entity ,fieldsToIndex); return batch.build(); } ); return ObservableTimer.time( batches, addTimer ); }
private void insertJsonBlob( String entityType, IndexEdge indexEdge, String filePath, final int max, final int startIndex ) throws IOException { InputStream is = this.getClass().getResourceAsStream( filePath ); ObjectMapper mapper = new ObjectMapper(); List<Object> sampleJson = mapper.readValue(is, new TypeReference<List<Object>>() { }); EntityIndexBatch batch = entityIndex.createBatch(); insertJsonBlob(sampleJson, batch, entityType, indexEdge, max, startIndex); indexProducer.put(batch.build()).subscribe();; EntityIndex.IndexRefreshCommandInfo info = entityIndex.refreshAsync().toBlocking().first(); long time = info.getExecutionTime(); logger.info("refresh took ms:" + time); }
@Override public Observable<FilterResult<Id>> call( final Observable<FilterResult<Candidate>> filterResultObservable ) { /** * A bit kludgy from old 1.0 -> 2.0 apis. Refactor this as we clean up our lower levels and create new results * objects */ final ApplicationScope applicationScope = pipelineContext.getApplicationScope(); final EntityCollectionManager entityCollectionManager = entityCollectionManagerFactory.createCollectionManager( applicationScope ); final EntityIndex applicationIndex = entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope)); final Observable<FilterResult<Id>> searchIdSetObservable = filterResultObservable.buffer( pipelineContext.getLimit() ).flatMap( candidateResults -> { //flatten toa list of ids to load final Observable<List<Id>> candidateIds = Observable.from( candidateResults ).map( candidate -> candidate.getValue().getCandidateResult().getId() ).toList(); //load the ids final Observable<VersionSet> versionSetObservable = candidateIds.flatMap( ids -> entityCollectionManager.getLatestVersion( ids ) ); //now we have a collection, validate our canidate set is correct. return versionSetObservable.map( entitySet -> new EntityCollector( applicationIndex.createBatch(), entitySet, candidateResults, indexProducer ) ).doOnNext( entityCollector -> entityCollector.merge() ).flatMap( entityCollector -> Observable.from( entityCollector.collectResults() ) ); } ); return searchIdSetObservable; }
@Override public Observable<IndexOperationMessage> indexEntity( final ApplicationScope applicationScope, final Entity entity ) { //bootstrap the lower modules from their caches final GraphManager gm = graphManagerFactory.createEdgeManager( applicationScope ); final EntityIndex ei = entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope)); final Id entityId = entity.getId(); //we always index in the target scope final Observable<Edge> edgesToTarget = edgesObservable.edgesToTarget( gm, entityId, true); //we may have to index we're indexing from source->target here final Observable<IndexEdge> sourceEdgesToIndex = edgesToTarget.map( edge -> generateScopeFromSource( edge ) ); //do our observable for batching //try to send a whole batch if we can final Observable<IndexOperationMessage> batches = sourceEdgesToIndex .buffer(indexFig.getIndexBatchSize() ) //map into batches based on our buffer size .flatMap( buffer -> Observable.from( buffer ) //collect results into a single batch .collect( () -> ei.createBatch(), ( batch, indexEdge ) -> { if (logger.isDebugEnabled()) { logger.debug("adding edge {} to batch for entity {}", indexEdge, entity); } final Optional<Set<String>> fieldsToIndex = getFilteredStringObjectMap( indexEdge ); batch.index( indexEdge, entity ,fieldsToIndex); } ) //return the future from the batch execution .map( batch -> batch.build() ) ); return ObservableTimer.time( batches, indexTimer ); }
EntityIndexBatch batch = entityIndex.createBatch(); insertJsonBlob( sampleJson, batch, entityType, indexEdge, size, 0 ); indexProducer.put(batch.build()).subscribe();;
@Override public Observable<IndexOperationMessage> deIndexOldVersions(final ApplicationScope applicationScope, final Id entityId, final List<UUID> versions) { final EntityIndex ei = entityIndexFactory. createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) ); // use LONG.MAX_VALUE in search edge because this value is not used elsewhere in lower code for de-indexing // previously .timsetamp() was used on entityId, but some entities do not have type-1 UUIDS ( legacy data) final SearchEdge searchEdgeFromSource = createSearchEdgeFromSource( new SimpleEdge( applicationScope.getApplication(), CpNamingUtils.getEdgeTypeFromCollectionName( InflectionUtils.pluralize( entityId.getType() ) ), entityId, Long.MAX_VALUE ) ); final EntityIndexBatch batch = ei.createBatch(); versions.forEach( version -> { batch.deindex(searchEdgeFromSource, entityId, version); }); return Observable.just(batch.build()); }
@Override public Observable<IndexOperationMessage> deleteIndexEdge( final ApplicationScope applicationScope, final Edge edge ) { final Observable<IndexOperationMessage> batches = Observable.just( edge ).map( edgeValue -> { final EntityIndex ei = entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) ); EntityIndexBatch batch = ei.createBatch(); //review why generating the Scope from the Source and the target node makes sense. final IndexEdge fromSource = generateScopeFromSource( edge ); final Id targetId = edge.getTargetNode(); CandidateResults targetEdgesToBeDeindexed = ei.getAllEdgeDocuments( fromSource, targetId ); //1. Feed the observable the candidate results you got back. Since it now does the aggregation for you // you don't need to worry about putting your code in a do while. batch = deindexBatchIteratorResolver( fromSource, targetEdgesToBeDeindexed, batch ); final IndexEdge fromTarget = generateScopeFromTarget( edge ); final Id sourceId = edge.getSourceNode(); CandidateResults sourceEdgesToBeDeindexed = ei.getAllEdgeDocuments( fromTarget, sourceId ); batch = deindexBatchIteratorResolver( fromTarget, sourceEdgesToBeDeindexed, batch ); return batch.build(); } ); return ObservableTimer.time( batches, addTimer ); }
@Test public void testDeleteWithAlias() throws IOException { final String entityType = "thing"; IndexEdge searchEdge = new IndexEdgeImpl( appId, "things", SearchEdge.NodeType.SOURCE, 1 ); final SearchTypes searchTypes = SearchTypes.fromTypes( entityType ); insertJsonBlob( entityType, searchEdge, "/sample-large.json", 1, 0 ); entityIndex.addIndex(UUID.randomUUID() + "v2", 1, 0, "one"); entityIndex.refreshAsync().toBlocking().first(); insertJsonBlob( entityType, searchEdge, "/sample-large.json", 1, 1 ); CandidateResults crs = testQuery( searchEdge, searchTypes, "name = 'Bowers Oneil'", 1 ); EntityIndexBatch entityIndexBatch = entityIndex.createBatch(); entityIndexBatch.deindex(searchEdge, crs.get(0)); indexProducer.put(entityIndexBatch.build()).subscribe(); entityIndex.refreshAsync().toBlocking().first(); //Hilda Youn testQuery(searchEdge, searchTypes, "name = 'Bowers Oneil'", 0); }
IndexEdge indexEdge = new IndexEdgeImpl( appId, "things", SearchEdge.NodeType.SOURCE, 1 ); final SearchTypes searchTypes = SearchTypes.fromTypes( entityType ); EntityIndexBatch batch = entityIndex.createBatch();
@Test public void testDeindex() { IndexEdge searchEdge = new IndexEdgeImpl( appId, "fastcars", SearchEdge.NodeType.SOURCE, 1 ); Map entityMap = new HashMap() {{ put( "name", "Ferrari 212 Inter" ); put( "introduced", 1952 ); put( "topspeed", 215 ); }}; Entity entity = EntityIndexMapUtils.fromMap( entityMap ); EntityUtils.setId(entity, new SimpleId( "fastcar" ) ); EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID() ); entity.setField(new UUIDField(IndexingUtils.ENTITY_ID_FIELDNAME, UUID.randomUUID() ) ); indexProducer.put(entityIndex.createBatch().index( searchEdge, entity ).build()).subscribe(); entityIndex.refreshAsync().toBlocking().first(); CandidateResults candidateResults = entityIndex .search( searchEdge, SearchTypes.fromTypes( entity.getId().getType() ), "name contains 'Ferrari*'", 10, 0, false ); assertEquals( 1, candidateResults.size() ); EntityIndexBatch batch = entityIndex.createBatch(); batch.deindex( searchEdge, entity ); indexProducer.put(batch.build()).subscribe();; entityIndex.refreshAsync().toBlocking().first(); candidateResults = entityIndex .search(searchEdge, SearchTypes.fromTypes( entity.getId().getType() ), "name contains 'Ferrari*'", 10, 0, false ); assertEquals(0, candidateResults.size()); }
@Test public void testSizeByEdge(){ final String type = UUID.randomUUID().toString(); Id ownerId = new SimpleId( "owner" ); final Entity first = new Entity( type ); first.setField( new StringField( "string", "I ate a sammich" ) ); first.setSize(100); EntityUtils.setVersion( first, UUIDGenerator.newTimeUUID() ); final Entity second = new Entity( type ); second.setSize(100); second.setField( new StringField( "string", "I drank a beer" ) ); EntityUtils.setVersion( second, UUIDGenerator.newTimeUUID() ); EntityIndexBatch batch = entityIndex.createBatch(); //get ordering, so 2 is before 1 when both match IndexEdge indexScope1 = new IndexEdgeImpl( ownerId,type , SearchEdge.NodeType.SOURCE, 10 ); batch.index( indexScope1, first ); IndexEdge indexScope2 = new IndexEdgeImpl( ownerId, type+"er", SearchEdge.NodeType.SOURCE, 11 ); batch.index( indexScope2, second); indexProducer.put(batch.build()).subscribe();; entityIndex.refreshAsync().toBlocking().first(); long size = entityIndex.getTotalEntitySizeInBytes(new SearchEdgeImpl(ownerId,type, SearchEdge.NodeType.SOURCE)); assertTrue( size == 100 ); }
final EntityIndexBatch batch = entityIndex.createBatch();
@Test public void deleteVerification() throws Throwable { Id ownerId = new SimpleId( "owner" ); IndexEdge indexSCope = new IndexEdgeImpl( ownerId, "user", SearchEdge.NodeType.SOURCE, 10 ); final String middleName = "middleName" + UUIDUtils.newTimeUUID(); Map entityMap = new HashMap() {{ put( "username", "edanuff" ); put( "email", "ed@anuff.com" ); put( "middlename", middleName ); }}; Entity user = EntityIndexMapUtils.fromMap( entityMap ); EntityUtils.setId( user, new SimpleId( "edanuff" ) ); EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() ); EntityIndexBatch batch = entityIndex.createBatch(); batch.index( indexSCope, user ); indexProducer.put(batch.build()).subscribe();; entityIndex.refreshAsync().toBlocking().first(); final String query = "where username = 'edanuff'"; CandidateResults r = entityIndex.search( indexSCope, SearchTypes.fromTypes( "edanuff" ), query, 10, 0, false); assertEquals( user.getId(), r.get( 0 ).getId()); batch.deindex( indexSCope, user.getId(), user.getVersion() ); indexProducer.put(batch.build()).subscribe();; entityIndex.refreshAsync().toBlocking().first(); // EntityRef r = entityIndex.search( indexSCope, SearchTypes.fromTypes( "edanuff" ), query, 10, 0, false ); assertFalse( r.iterator().hasNext() ); }
@Test public void queryByUUID() throws Throwable { Id appId = new SimpleId( "application" ); Id ownerId = new SimpleId( "owner" ); IndexEdge indexSCope = new IndexEdgeImpl( ownerId, "user", SearchEdge.NodeType.SOURCE, 10 ); final UUID searchUUID = UUIDGenerator.newTimeUUID(); Map entityMap = new HashMap() {{ put( "searchUUID", searchUUID ); }}; Entity user = EntityIndexMapUtils.fromMap( entityMap ); final Id entityId = new SimpleId( "entitytype" ); EntityUtils.setId( user, entityId ); EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() ); EntityIndexBatch batch = entityIndex.createBatch(); batch.index( indexSCope, user ); indexProducer.put(batch.build()).subscribe();; entityIndex.refreshAsync().toBlocking().first(); final String query = "where searchUUID = " + searchUUID; final CandidateResults r = entityIndex.search( indexSCope, SearchTypes.fromTypes(entityId.getType()), query, 10, 0, false); assertEquals(user.getId(), r.get(0).getId()); }
@Test public void queryByStringWildCardSpaces() throws Throwable { Id appId = new SimpleId( "application" ); Id ownerId = new SimpleId( "owner" ); IndexEdge indexSCope = new IndexEdgeImpl( ownerId, "user", SearchEdge.NodeType.SOURCE, 10 ); Map entityMap = new HashMap() {{ put( "string", "I am a search string" ); }}; Entity user = EntityIndexMapUtils.fromMap( entityMap ); final Id entityId = new SimpleId( "entitytype" ); EntityUtils.setId( user, entityId ); EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() ); EntityIndexBatch batch = entityIndex.createBatch(); batch.index(indexSCope, user); indexProducer.put(batch.build()).subscribe();; entityIndex.refreshAsync().toBlocking().first(); final String query = "where string = 'I am*'"; final CandidateResults r = entityIndex.search( indexSCope, SearchTypes.fromTypes( entityId.getType() ), query, 10, 0, false); assertEquals(user.getId(), r.get(0).getId()); //shouldn't match final String queryNoWildCard = "where string = 'I am'"; final CandidateResults noWildCardResults = entityIndex.search( indexSCope, SearchTypes.fromTypes( entityId.getType() ), queryNoWildCard, 10, 0, false ); assertEquals( 0, noWildCardResults.size() ); }
final EntityIndexBatch batch = entityIndex.createBatch();
EntityIndexBatch batch = entityIndex.createBatch(); batch.index(indexSCope, first ); batch.index( indexSCope, second );
EntityIndexBatch batch = entityIndex.createBatch();
EntityIndexBatch batch = entityIndex.createBatch();