@Override public void addIndex(final String newIndexName,final int shards,final int replicas, final String writeConsistency){ managerCache.getEntityIndex(applicationScope).addIndex(newIndexName, shards, replicas, writeConsistency); }
@Override public Observable<IndexOperationMessage> deIndexEdge(final ApplicationScope applicationScope, final Edge edge, final Id entityId, final UUID entityVersion){ if (logger.isTraceEnabled()) { logger.trace("deIndexEdge edge={} entityId={} entityVersion={}", edge.toString(), entityId.toString(), entityVersion.toString()); } final EntityIndex ei = entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope)); final EntityIndexBatch entityBatch = ei.createBatch(); entityBatch.deindex(generateScopeFromSource( edge ), entityId, entityVersion); return Observable.just(entityBatch.build()); }
@Test public void healthTest() { Id appId = new SimpleId( "entityindextest" ); assertNotEquals( "cluster should be ok", Health.RED, entityIndex.getClusterHealth() ); assertEquals( "index should be ready", Health.GREEN, entityIndex.getIndexHealth() ); entityIndex.refreshAsync().toBlocking().first(); assertNotEquals( "cluster should be fine", Health.RED, entityIndex.getIndexHealth() ); assertNotEquals( "cluster should be ready now", Health.RED, entityIndex.getClusterHealth() ); }
@Override public Observable<IndexOperationMessage> deleteIndexEdge( final ApplicationScope applicationScope, final Edge edge ) { final Observable<IndexOperationMessage> batches = Observable.just( edge ).map( edgeValue -> { final EntityIndex ei = entityIndexFactory.createEntityIndex(indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope) ); EntityIndexBatch batch = ei.createBatch(); //review why generating the Scope from the Source and the target node makes sense. final IndexEdge fromSource = generateScopeFromSource( edge ); final Id targetId = edge.getTargetNode(); CandidateResults targetEdgesToBeDeindexed = ei.getAllEdgeDocuments( fromSource, targetId ); //1. Feed the observable the candidate results you got back. Since it now does the aggregation for you // you don't need to worry about putting your code in a do while. batch = deindexBatchIteratorResolver( fromSource, targetEdgesToBeDeindexed, batch ); final IndexEdge fromTarget = generateScopeFromTarget( edge ); final Id sourceId = edge.getSourceNode(); CandidateResults sourceEdgesToBeDeindexed = ei.getAllEdgeDocuments( fromTarget, sourceId ); batch = deindexBatchIteratorResolver( fromTarget, sourceEdgesToBeDeindexed, batch ); return batch.build(); } ); return ObservableTimer.time( batches, addTimer ); }
private void insertJsonBlob( String entityType, IndexEdge indexEdge, String filePath, final int max, final int startIndex ) throws IOException { InputStream is = this.getClass().getResourceAsStream( filePath ); ObjectMapper mapper = new ObjectMapper(); List<Object> sampleJson = mapper.readValue(is, new TypeReference<List<Object>>() { }); EntityIndexBatch batch = entityIndex.createBatch(); insertJsonBlob(sampleJson, batch, entityType, indexEdge, max, startIndex); indexProducer.put(batch.build()).subscribe();; EntityIndex.IndexRefreshCommandInfo info = entityIndex.refreshAsync().toBlocking().first(); long time = info.getExecutionTime(); logger.info("refresh took ms:" + time); }
@Test public void testDeindex() { IndexEdge searchEdge = new IndexEdgeImpl( appId, "fastcars", SearchEdge.NodeType.SOURCE, 1 ); Map entityMap = new HashMap() {{ put( "name", "Ferrari 212 Inter" ); put( "introduced", 1952 ); put( "topspeed", 215 ); }}; Entity entity = EntityIndexMapUtils.fromMap( entityMap ); EntityUtils.setId(entity, new SimpleId( "fastcar" ) ); EntityUtils.setVersion(entity, UUIDGenerator.newTimeUUID() ); entity.setField(new UUIDField(IndexingUtils.ENTITY_ID_FIELDNAME, UUID.randomUUID() ) ); indexProducer.put(entityIndex.createBatch().index( searchEdge, entity ).build()).subscribe(); entityIndex.refreshAsync().toBlocking().first(); CandidateResults candidateResults = entityIndex .search( searchEdge, SearchTypes.fromTypes( entity.getId().getType() ), "name contains 'Ferrari*'", 10, 0, false ); assertEquals( 1, candidateResults.size() ); EntityIndexBatch batch = entityIndex.createBatch(); batch.deindex( searchEdge, entity ); indexProducer.put(batch.build()).subscribe();; entityIndex.refreshAsync().toBlocking().first(); candidateResults = entityIndex .search(searchEdge, SearchTypes.fromTypes( entity.getId().getType() ), "name contains 'Ferrari*'", 10, 0, false ); assertEquals(0, candidateResults.size()); }
private CandidateResults getResults( final EntityIndex entityIndex, final SearchEdge searchEdge, final SearchTypes searchTypes, final int expectedSize, final int attempts ) { for ( int i = 0; i < attempts; i++ ) { final CandidateResults candidateResults = entityIndex.search( searchEdge, searchTypes, "select *", 100, 0, false ); if ( candidateResults.size() == expectedSize ) { return candidateResults; } try { Thread.sleep( 10000 ); } catch ( InterruptedException e ) { //swallow } } fail( "Could not find candidates of size " + expectedSize + " after " + attempts + " attempts" ); //we'll never reach this, required for compile return null; } }
@Test public void testDeleteWithAlias() throws IOException { final String entityType = "thing"; IndexEdge searchEdge = new IndexEdgeImpl( appId, "things", SearchEdge.NodeType.SOURCE, 1 ); final SearchTypes searchTypes = SearchTypes.fromTypes( entityType ); insertJsonBlob( entityType, searchEdge, "/sample-large.json", 1, 0 ); entityIndex.addIndex(UUID.randomUUID() + "v2", 1, 0, "one"); entityIndex.refreshAsync().toBlocking().first(); insertJsonBlob( entityType, searchEdge, "/sample-large.json", 1, 1 ); CandidateResults crs = testQuery( searchEdge, searchTypes, "name = 'Bowers Oneil'", 1 ); EntityIndexBatch entityIndexBatch = entityIndex.createBatch(); entityIndexBatch.deindex(searchEdge, crs.get(0)); indexProducer.put(entityIndexBatch.build()).subscribe(); entityIndex.refreshAsync().toBlocking().first(); //Hilda Youn testQuery(searchEdge, searchTypes, "name = 'Bowers Oneil'", 0); }
@Test public void testSizeByEdge(){ final String type = UUID.randomUUID().toString(); Id ownerId = new SimpleId( "owner" ); final Entity first = new Entity( type ); first.setField( new StringField( "string", "I ate a sammich" ) ); first.setSize(100); EntityUtils.setVersion( first, UUIDGenerator.newTimeUUID() ); final Entity second = new Entity( type ); second.setSize(100); second.setField( new StringField( "string", "I drank a beer" ) ); EntityUtils.setVersion( second, UUIDGenerator.newTimeUUID() ); EntityIndexBatch batch = entityIndex.createBatch(); //get ordering, so 2 is before 1 when both match IndexEdge indexScope1 = new IndexEdgeImpl( ownerId,type , SearchEdge.NodeType.SOURCE, 10 ); batch.index( indexScope1, first ); IndexEdge indexScope2 = new IndexEdgeImpl( ownerId, type+"er", SearchEdge.NodeType.SOURCE, 11 ); batch.index( indexScope2, second); indexProducer.put(batch.build()).subscribe();; entityIndex.refreshAsync().toBlocking().first(); long size = entityIndex.getTotalEntitySizeInBytes(new SearchEdgeImpl(ownerId,type, SearchEdge.NodeType.SOURCE)); assertTrue( size == 100 ); }
@Override public void initializeManagementIndex(){ getManagementIndex().initialize(); } }
final Observable deleteAppFromIndex = aei.deleteApplication(); ei.refreshAsync().toBlocking().last(); } catch (Exception e) { throw new RuntimeException(e);
@Override public long getSize(ApplicationScope applicationScope, SearchEdge edge) { final IndexLocationStrategy indexLocationStrategy = indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope); EntityIndex entityIndex = entityIndexFactory.createEntityIndex(indexLocationStrategy); return entityIndex.getTotalEntitySizeInBytes(edge); }
@Override public void refreshIndex() { this.entityIndex = entityIndex == null ? injector.getInstance(EntityIndex.class) : entityIndex; entityIndex.refreshAsync().toBlocking().first(); } }
@Override public Health getIndexHealth() { return getManagementIndex().getIndexHealth(); }
/** * Delete app index */ private void deleteIndex(UUID appUuid) { Injector injector = SpringResource.getInstance().getBean(Injector.class); IndexLocationStrategyFactory indexLocationStrategyFactory = injector.getInstance(IndexLocationStrategyFactory.class); EntityIndexFactory eif = injector.getInstance(EntityIndexFactory.class); Id appId = new SimpleId(appUuid, Schema.TYPE_APPLICATION); ApplicationScope scope = new ApplicationScopeImpl(appId); EntityIndex ei = eif.createEntityIndex( indexLocationStrategyFactory.getIndexLocationStrategy(scope) ); ei.deleteApplication().toBlocking().lastOrDefault(null); app.waitForQueueDrainAndRefreshIndex(); }
@Test public void deleteVerification() throws Throwable { Id ownerId = new SimpleId( "owner" ); IndexEdge indexSCope = new IndexEdgeImpl( ownerId, "user", SearchEdge.NodeType.SOURCE, 10 ); final String middleName = "middleName" + UUIDUtils.newTimeUUID(); Map entityMap = new HashMap() {{ put( "username", "edanuff" ); put( "email", "ed@anuff.com" ); put( "middlename", middleName ); }}; Entity user = EntityIndexMapUtils.fromMap( entityMap ); EntityUtils.setId( user, new SimpleId( "edanuff" ) ); EntityUtils.setVersion( user, UUIDGenerator.newTimeUUID() ); EntityIndexBatch batch = entityIndex.createBatch(); batch.index( indexSCope, user ); indexProducer.put(batch.build()).subscribe();; entityIndex.refreshAsync().toBlocking().first(); final String query = "where username = 'edanuff'"; CandidateResults r = entityIndex.search( indexSCope, SearchTypes.fromTypes( "edanuff" ), query, 10, 0, false); assertEquals( user.getId(), r.get( 0 ).getId()); batch.deindex( indexSCope, user.getId(), user.getVersion() ); indexProducer.put(batch.build()).subscribe();; entityIndex.refreshAsync().toBlocking().first(); // EntityRef r = entityIndex.search( indexSCope, SearchTypes.fromTypes( "edanuff" ), query, 10, 0, false ); assertFalse( r.iterator().hasNext() ); }
private CandidateResults getResults( final EntityIndex EntityIndex, final SearchEdge searchEdge, final SearchTypes searchTypes, final int expectedSize ) { final int attempts = 100; String ql = "select *"; for ( int i = 0; i < attempts; i++ ) { final CandidateResults candidateResults = EntityIndex.search( searchEdge, searchTypes, ql , 100, 0, false ); if ( candidateResults.size() == expectedSize ) { return candidateResults; } try { Thread.sleep( 100 ); } catch ( InterruptedException e ) { //swallow } } fail( "Could not find candidates of size " + expectedSize + "after " + attempts + " attempts" ); //we'll never reach this, required for compile return null; } }
public EntityIndex load( IndexLocationStrategy locationStrategy ) { EntityIndex index = new EsEntityIndexImpl( provider, indexCache, config, metricsFactory, locationStrategy ); index.initialize(); return index; } } );
@Override public long getApplicationSize(ApplicationScope applicationScope) { final IndexLocationStrategy indexLocationStrategy = indexLocationStrategyFactory.getIndexLocationStrategy(applicationScope); EntityIndex entityIndex = entityIndexFactory.createEntityIndex(indexLocationStrategy); GraphManager graphManager = graphManagerFactory.createEdgeManager(applicationScope); Long sum = ObservableTimer.time( MathObservable.sumLong( graphManager.getEdgeTypesFromSource(new SimpleSearchEdgeType(applicationScope.getApplication(), CpNamingUtils.EDGE_COLL_PREFIX, Optional.<String>absent())) .map(type -> CpNamingUtils.createCollectionSearchEdge(applicationScope.getApplication(), type)) .map(edge -> entityIndex.getTotalEntitySizeInBytes(edge)) ), sumTimer).toBlocking().last(); return sum.longValue(); }
public EntityIndex.IndexRefreshCommandInfo refreshIndex() { try { return managerCache.getEntityIndex(applicationScope).refreshAsync().toBlocking().first(); } catch (Exception e) { throw new RuntimeException("refresh failed",e); } }