@Override public EntityIndexBatch deindex(final SearchEdge searchEdge, final CandidateResult entity) { return deindex( searchEdge, entity.getId(), entity.getVersion() ); } @Override
/** * Merge our candidates and our entity set into results */ public void merge(boolean keepStaleEntries, String query, boolean isDirectQuery) { if (!isDirectQuery) { filterDuplicateCandidates(query); } else { // remove direct query duplicates or missing entities (names that don't exist will have null ids) Set<UUID> foundUUIDs = new HashSet<>(); for (FilterResult<Candidate> candidateFilterResult : candidateResults) { Id id = candidateFilterResult.getValue().getCandidateResult().getId(); if (id != null) { UUID uuid = id.getUuid(); if (!foundUUIDs.contains(uuid)) { dedupedCandidateResults.add(candidateFilterResult); foundUUIDs.add(uuid); } } } } for (final FilterResult<Candidate> candidateResult : dedupedCandidateResults) { validate(candidateResult, keepStaleEntries, query, isDirectQuery); } // no index requests made for direct query, so no need to modify index if (!isDirectQuery) { indexProducer.put(batch.build()).toBlocking().lastOrDefault(null); // want to rethrow if batch fails } }
@Override public Observable<FilterResult<Id>> call( final Observable<FilterResult<Candidate>> filterResultObservable ) { /** * A bit kludgy from old 1.0 -> 2.0 apis. Refactor this as we clean up our lower levels and create new results * objects */ final ApplicationScope applicationScope = pipelineContext.getApplicationScope(); final EntityCollectionManager entityCollectionManager = entityCollectionManagerFactory.createCollectionManager( applicationScope ); final EntityIndex applicationIndex = entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope)); final Observable<FilterResult<Id>> searchIdSetObservable = filterResultObservable.buffer( pipelineContext.getLimit() ).flatMap( candidateResults -> { //flatten toa list of ids to load final Observable<List<Id>> candidateIds = Observable.from( candidateResults ).map( candidate -> candidate.getValue().getCandidateResult().getId() ).toList(); //load the ids final Observable<VersionSet> versionSetObservable = candidateIds.flatMap( ids -> entityCollectionManager.getLatestVersion( ids ) ); //now we have a collection, validate our canidate set is correct. return versionSetObservable.map( entitySet -> new EntityCollector( applicationIndex.createBatch(), entitySet, candidateResults, indexProducer ) ).doOnNext( entityCollector -> entityCollector.merge() ).flatMap( entityCollector -> Observable.from( entityCollector.collectResults() ) ); } ); return searchIdSetObservable; }
final Candidate candidate = filterResult.getValue(); final CandidateResult candidateResult = candidate.getCandidateResult(); final Id candidateId = candidateResult.getId(); final UUID candidateVersion = candidateResult.getVersion(); final Candidate candidate = filterResult.getValue(); final CandidateResult candidateResult = candidate.getCandidateResult(); final Id candidateId = candidateResult.getId(); final UUID candidateVersion = candidateResult.getVersion();
/** * Validate each candidate results vs the data loaded from cass */ private void validate( final FilterResult<Candidate> filterCandidate ) { final CandidateResult candidateResult = filterCandidate.getValue().getCandidateResult(); final SearchEdge searchEdge = filterCandidate.getValue().getSearchEdge(); final MvccLogEntry logEntry = versionSet.getMaxVersion( candidateResult.getId() ); final UUID candidateVersion = candidateResult.getVersion(); final UUID entityVersion = logEntry.getVersion(); final Id entityId = logEntry.getEntityId(); //entity is newer than ES version if ( UUIDComparator.staticCompare( entityVersion, candidateVersion ) > 0 ) { logger.warn( "Deindexing stale entity on edge {} for entityId {} and version {}", searchEdge, entityId, entityVersion ); batch.deindex( searchEdge, entityId, entityVersion ); return; } //ES is newer than cass, it means we haven't repaired the record in Cass, we don't want to //remove the ES record, since the read in cass should cause a read repair, just ignore if ( UUIDComparator.staticCompare( candidateVersion, entityVersion ) > 0 ) { logger.warn( "Found a newer version in ES over cassandra for edge {} for entityId {} and version {}. Repair should be run", searchEdge, entityId, entityVersion ); } //they're the same add it final FilterResult<Id> result = new FilterResult<>( entityId, filterCandidate.getPath() ); results.add( result ); } }
assertEquals( entity1.getId(), candidate1.getId() ); assertEquals(entity1.getVersion(), candidate1.getVersion());
@Test public void deleteVerification() throws Throwable { Id ownerId = new SimpleId( "owner" ); IndexEdge indexSCope = new IndexEdgeImpl( ownerId, "user", SearchEdge.NodeType.SOURCE, 10 ); final String middleName = "middleName" + UUIDUtils.newTimeUUID(); Map entityMap = new HashMap() {{ put( "username", "edanuff" ); put( "email", "ed@anuff.com" ); put( "middlename", middleName ); }}; Entity user = EntityIndexMapUtils.fromMap( entityMap ); EntityUtils.setId( user, new SimpleId( "edanuff" ) ); EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() ); EntityIndexBatch batch = entityIndex.createBatch(); batch.index( indexSCope, user ); indexProducer.put(batch.build()).subscribe();; entityIndex.refreshAsync().toBlocking().first(); final String query = "where username = 'edanuff'"; CandidateResults r = entityIndex.search( indexSCope, SearchTypes.fromTypes( "edanuff" ), query, 10, 0, false); assertEquals( user.getId(), r.get( 0 ).getId()); batch.deindex( indexSCope, user.getId(), user.getVersion() ); indexProducer.put(batch.build()).subscribe();; entityIndex.refreshAsync().toBlocking().first(); // EntityRef r = entityIndex.search( indexSCope, SearchTypes.fromTypes( "edanuff" ), query, 10, 0, false ); assertFalse( r.iterator().hasNext() ); }
assertEquals(first.getId(), singleResults.get(0).getId()); assertEquals( second.getId(), singleKeywordUnion.get( 0).getId() ); assertEquals( first.getId(), singleKeywordUnion.get( 1 ).getId() ); assertEquals(second.getId(), towMatchResults.get( 0).getId() ); assertEquals(first.getId(), towMatchResults.get( 1 ).getId() );
@Test public void queryByUUID() throws Throwable { Id appId = new SimpleId( "application" ); Id ownerId = new SimpleId( "owner" ); IndexEdge indexSCope = new IndexEdgeImpl( ownerId, "user", SearchEdge.NodeType.SOURCE, 10 ); final UUID searchUUID = UUIDGenerator.newTimeUUID(); Map entityMap = new HashMap() {{ put( "searchUUID", searchUUID ); }}; Entity user = EntityIndexMapUtils.fromMap( entityMap ); final Id entityId = new SimpleId( "entitytype" ); EntityUtils.setId( user, entityId ); EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() ); EntityIndexBatch batch = entityIndex.createBatch(); batch.index( indexSCope, user ); indexProducer.put(batch.build()).subscribe();; entityIndex.refreshAsync().toBlocking().first(); final String query = "where searchUUID = " + searchUUID; final CandidateResults r = entityIndex.search( indexSCope, SearchTypes.fromTypes(entityId.getType()), query, 10, 0, false); assertEquals(user.getId(), r.get(0).getId()); }
assertEquals(second.getId(), notFirstResults.get( 0 ).getId() ); assertEquals( first.getId(), notSecondUnion.get( 0 ).getId() ); assertEquals( second.getId(), notBothReturnResults.get( 0).getId() ); assertEquals( first.getId(), notBothReturnResults.get( 1 ).getId() ); assertEquals( second.getId(), noMatchesAndResults.get( 0).getId() ); assertEquals( first.getId(), noMatchesAndResults.get( 1 ).getId() ); assertEquals( second.getId(), noMatchesOrResults.get( 0).getId() ); assertEquals( first.getId(), noMatchesOrResults.get( 1 ).getId() );
@Test public void queryByStringWildCardSpaces() throws Throwable { Id appId = new SimpleId( "application" ); Id ownerId = new SimpleId( "owner" ); IndexEdge indexSCope = new IndexEdgeImpl( ownerId, "user", SearchEdge.NodeType.SOURCE, 10 ); Map entityMap = new HashMap() {{ put( "string", "I am a search string" ); }}; Entity user = EntityIndexMapUtils.fromMap( entityMap ); final Id entityId = new SimpleId( "entitytype" ); EntityUtils.setId( user, entityId ); EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() ); EntityIndexBatch batch = entityIndex.createBatch(); batch.index(indexSCope, user); indexProducer.put(batch.build()).subscribe();; entityIndex.refreshAsync().toBlocking().first(); final String query = "where string = 'I am*'"; final CandidateResults r = entityIndex.search( indexSCope, SearchTypes.fromTypes( entityId.getType() ), query, 10, 0, false); assertEquals(user.getId(), r.get(0).getId()); //shouldn't match final String queryNoWildCard = "where string = 'I am'"; final CandidateResults noWildCardResults = entityIndex.search( indexSCope, SearchTypes.fromTypes( entityId.getType() ), queryNoWildCard, 10, 0, false ); assertEquals( 0, noWildCardResults.size() ); }
assertEquals(expected.getId(), candidate.getId());
@Test public void testDocumentId() { final ApplicationScopeImpl applicationScope = new ApplicationScopeImpl( new SimpleId( "application" ) ); final Id id = new SimpleId( "id" ); final UUID version = UUIDGenerator.newTimeUUID(); final SearchEdgeImpl searchEdge = new SearchEdgeImpl( new SimpleId( "source" ), "users", SearchEdge.NodeType.TARGET ); final String output = IndexingUtils.createIndexDocId( applicationScope, id, version, searchEdge ); final String expected = "appId(" + applicationScope.getApplication().getUuid() + ",application).entityId(" + id.getUuid() + "," + id .getType() + ").version(" + version + ").nodeId(" + searchEdge.getNodeId().getUuid() + "," + searchEdge .getNodeId().getType() + ").edgeName(users).nodeType(TARGET)"; assertEquals( output, expected ); //now parse it final CandidateResult parsedId = parseIndexDocId( output ); assertEquals(version, parsedId.getVersion()); assertEquals(id, parsedId.getId()); }
@Test public void testDocumentIdPipes() { final ApplicationScopeImpl applicationScope = new ApplicationScopeImpl( new SimpleId( "application" ) ); final Id id = new SimpleId( "id" ); final UUID version = UUIDGenerator.newTimeUUID(); final SearchEdgeImpl searchEdge = new SearchEdgeImpl( new SimpleId( "source" ), "zzzcollzzz|users", SearchEdge.NodeType.TARGET ); final String output = IndexingUtils.createIndexDocId( applicationScope, id, version, searchEdge ); final String expected = "appId(" + applicationScope.getApplication().getUuid() + ",application).entityId(" + id.getUuid() + "," + id .getType() + ").version(" + version + ").nodeId(" + searchEdge.getNodeId().getUuid() + "," + searchEdge .getNodeId().getType() + ").edgeName(zzzcollzzz|users).nodeType(TARGET)"; assertEquals( output, expected ); //now parse it final CandidateResult parsedId = parseIndexDocId( output ); assertEquals(version, parsedId.getVersion()); assertEquals(id, parsedId.getId()); }
@Test public void testAppIdFromDocumentId() { final ApplicationScopeImpl applicationScope = new ApplicationScopeImpl( new SimpleId( "application" ) ); final Id id = new SimpleId( "id" ); final UUID version = UUIDGenerator.newTimeUUID(); final SearchEdgeImpl searchEdge = new SearchEdgeImpl( new SimpleId( "source" ), "users", SearchEdge.NodeType.TARGET ); final String output = IndexingUtils.createIndexDocId( applicationScope, id, version, searchEdge ); final String expected = "appId(" + applicationScope.getApplication().getUuid() + ",application).entityId(" + id.getUuid() + "," + id .getType() + ").version(" + version + ").nodeId(" + searchEdge.getNodeId().getUuid() + "," + searchEdge .getNodeId().getType() + ").edgeName(users).nodeType(TARGET)"; assertEquals( output, expected ); //now parse it final CandidateResult parsedId = parseIndexDocId( output ); assertEquals(version, parsedId.getVersion()); assertEquals(id, parsedId.getId()); final UUID appId = parseAppIdFromIndexDocId(output); assertEquals(appId,applicationScope.getApplication().getUuid()); }