@VisibleForTesting void index(ComponentDto... docs) { BulkIndexer bulk = new BulkIndexer(esClient, INDEX_TYPE_COMPONENT, Size.REGULAR); bulk.start(); Arrays.stream(docs) .map(ComponentIndexer::toDocument) .map(ComponentIndexer::newIndexRequest) .forEach(bulk::add); bulk.stop(); }
public void deleteByKeys(String projectUuid, Collection<String> issueKeys) { if (issueKeys.isEmpty()) { return; } BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, IndexingListener.FAIL_ON_ERROR); bulkIndexer.start(); issueKeys.forEach(issueKey -> bulkIndexer.addDeletion(INDEX_TYPE_ISSUE, issueKey, projectUuid)); bulkIndexer.stop(); }
public void delete(String projectUuid, Collection<String> disabledComponentUuids) { BulkIndexer bulk = new BulkIndexer(esClient, INDEX_TYPE_COMPONENT, Size.REGULAR); bulk.start(); disabledComponentUuids.forEach(uuid -> bulk.addDeletion(INDEX_TYPE_COMPONENT, uuid, projectUuid)); bulk.stop(); }
private void doIndex(Iterator<IssueDoc> issues, Size size, IndexingListener listener) { BulkIndexer bulk = createBulkIndexer(size, listener); bulk.start(); while (issues.hasNext()) { IssueDoc issue = issues.next(); bulk.add(newIndexRequest(issue)); } bulk.stop(); }
private void index(Collection<IndexPermissions> authorizations, Stream<AuthorizationScope> scopes, Size bulkSize) { if (authorizations.isEmpty()) { return; } // index each authorization in each scope scopes.forEach(scope -> { IndexType indexType = scope.getIndexType(); BulkIndexer bulkIndexer = new BulkIndexer(esClient, indexType, bulkSize); bulkIndexer.start(); authorizations.stream() .filter(scope.getProjectPredicate()) .map(dto -> newIndexRequest(dto, indexType)) .forEach(bulkIndexer::add); bulkIndexer.stop(); }); }
/** * Index a single document. * <p/> * The views lookup cache will be cleared */ public void index(ViewDoc viewDoc) { BulkIndexer bulk = new BulkIndexer(esClient, ViewIndexDefinition.INDEX_TYPE_VIEW, Size.REGULAR); bulk.start(); doIndex(bulk, viewDoc, true); bulk.stop(); }
/** * Delete all the documents matching the given search request. This method is blocking. * Index is refreshed, so docs are not searchable as soon as method is executed. * <p> * Note that the parameter indexType could be removed if progress logs are not needed. */ public static IndexingResult delete(EsClient client, IndexType indexType, SearchRequestBuilder searchRequest) { BulkIndexer bulk = new BulkIndexer(client, indexType, Size.REGULAR); bulk.start(); bulk.addDeletion(searchRequest); return bulk.stop(); }
private void index(DbSession dbSession, Map<String, String> viewAndProjectViewUuidMap, boolean needClearCache, Size bulkSize) { BulkIndexer bulk = new BulkIndexer(esClient, ViewIndexDefinition.INDEX_TYPE_VIEW, bulkSize); bulk.start(); for (Map.Entry<String, String> entry : viewAndProjectViewUuidMap.entrySet()) { String viewUuid = entry.getKey(); List<String> projects = dbClient.componentDao().selectProjectsFromView(dbSession, viewUuid, entry.getValue()); doIndex(bulk, new ViewDoc() .setUuid(viewUuid) .setProjects(projects), needClearCache); } bulk.stop(); }
private IndexingResult doIndexActiveRules(DbSession dbSession, Map<Long, EsQueueDto> activeRuleItems) { OneToOneResilientIndexingListener listener = new OneToOneResilientIndexingListener(dbClient, dbSession, activeRuleItems.values()); BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, listener); bulkIndexer.start(); Map<Long, EsQueueDto> remaining = new HashMap<>(activeRuleItems); dbClient.activeRuleDao().scrollByIdsForIndexing(dbSession, activeRuleItems.keySet(), i -> { remaining.remove(i.getId()); bulkIndexer.add(newIndexRequest(i)); }); // the remaining ids reference rows that don't exist in db. They must // be deleted from index. remaining.values().forEach(item -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE, item.getDocId(), item.getDocRouting())); return bulkIndexer.stop(); }
private void doIndex(Size size, @Nullable String projectUuid) { try (DbSession dbSession = dbClient.openSession(false); ProjectMeasuresIndexerIterator rowIt = ProjectMeasuresIndexerIterator.create(dbSession, projectUuid)) { BulkIndexer bulkIndexer = createBulkIndexer(size, IndexingListener.FAIL_ON_ERROR); bulkIndexer.start(); while (rowIt.hasNext()) { ProjectMeasures doc = rowIt.next(); bulkIndexer.add(newIndexRequest(toProjectMeasuresDoc(doc))); } bulkIndexer.stop(); } }
@Override public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { try (DbSession dbSession = dbClient.openSession(false)) { ListMultimap<String, String> organizationUuidsByUserUuid = ArrayListMultimap.create(); dbClient.organizationMemberDao().selectAllForUserIndexing(dbSession, organizationUuidsByUserUuid::put); BulkIndexer bulkIndexer = newBulkIndexer(Size.LARGE, IndexingListener.FAIL_ON_ERROR); bulkIndexer.start(); dbClient.userDao().scrollAll(dbSession, // only index requests, no deletion requests. // Deactivated users are not deleted but updated. u -> bulkIndexer.add(newIndexRequest(u, organizationUuidsByUserUuid))); bulkIndexer.stop(); } }
@Override public void indexOnStartup(Set<IndexType> uninitializedIndexTypes) { try (DbSession dbSession = dbClient.openSession(false)) { BulkIndexer bulkIndexer = createBulkIndexer(Size.LARGE, IndexingListener.FAIL_ON_ERROR); bulkIndexer.start(); dbClient.activeRuleDao().scrollAllForIndexing(dbSession, ar -> bulkIndexer.add(newIndexRequest(ar))); bulkIndexer.stop(); } }
@Test public void index_nothing() { BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR); indexer.start(); indexer.stop(); assertThat(count()).isEqualTo(0); }
/** * @param projectUuid the uuid of the project to analyze, or {@code null} if all content should be indexed.<br/> * <b>Warning:</b> only use {@code null} during startup. */ private void doIndexByProjectUuid(@Nullable String projectUuid, Size bulkSize) { BulkIndexer bulk = new BulkIndexer(esClient, INDEX_TYPE_COMPONENT, bulkSize); bulk.start(); try (DbSession dbSession = dbClient.openSession(false)) { dbClient.componentDao() .scrollForIndexing(dbSession, projectUuid, context -> { ComponentDto dto = context.getResultObject(); bulk.add(newIndexRequest(toDocument(dto))); }); } bulk.stop(); }
@Test public void listener_is_not_called_with_errors() { FakeListener listener = new FakeListener(); BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR, listener); indexer.start(); indexer.add(newIndexRequestWithDocId("foo")); indexer.add(new IndexRequest("index_does_not_exist", "index_does_not_exist", "bar").source(emptyMap())); indexer.stop(); assertThat(listener.calledDocIds).containsExactly(new DocId(INDEX_TYPE_FAKE, "foo")); assertThat(listener.calledResult.getSuccess()).isEqualTo(1); assertThat(listener.calledResult.getTotal()).isEqualTo(2); }
@Test public void listener_is_called_on_successful_requests() { FakeListener listener = new FakeListener(); BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR, listener); indexer.start(); indexer.addDeletion(INDEX_TYPE_FAKE, "foo"); indexer.stop(); assertThat(listener.calledDocIds) .containsExactlyInAnyOrder(new DocId(INDEX_TYPE_FAKE, "foo")); assertThat(listener.calledResult.getSuccess()).isEqualTo(1); assertThat(listener.calledResult.getTotal()).isEqualTo(1); }
@Test public void index_documents() { BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR); indexer.start(); indexer.add(newIndexRequest(42)); indexer.add(newIndexRequest(78)); // request is not sent yet assertThat(count()).isEqualTo(0); // send remaining requests indexer.stop(); assertThat(count()).isEqualTo(2); }
@Test public void listener_is_called_even_if_deleting_a_doc_that_does_not_exist() { FakeListener listener = new FakeListener(); BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR, listener); indexer.start(); indexer.add(newIndexRequestWithDocId("foo")); indexer.add(newIndexRequestWithDocId("bar")); indexer.stop(); assertThat(listener.calledDocIds) .containsExactlyInAnyOrder(new DocId(INDEX_TYPE_FAKE, "foo"), new DocId(INDEX_TYPE_FAKE, "bar")); assertThat(listener.calledResult.getSuccess()).isEqualTo(2); assertThat(listener.calledResult.getTotal()).isEqualTo(2); }
@Test public void log_requests_when_TRACE_level_is_enabled() { logTester.setLevel(LoggerLevel.TRACE); BulkIndexer indexer = new BulkIndexer(es.client(), INDEX_TYPE_FAKE, Size.REGULAR, new FakeListener()); indexer.start(); indexer.add(newIndexRequestWithDocId("foo")); indexer.addDeletion(INDEX_TYPE_FAKE, "foo"); indexer.add(newIndexRequestWithDocId("bar")); indexer.stop(); assertThat(logTester.logs(LoggerLevel.TRACE) .stream() .filter(log -> log.contains("Bulk[2 index requests on fakes/fake, 1 delete requests on fakes/fake]")) .count()).isNotZero(); }
private IndexingResult doIndexIssueItems(DbSession dbSession, ListMultimap<String, EsQueueDto> itemsByIssueKey) { if (itemsByIssueKey.isEmpty()) { return new IndexingResult(); } IndexingListener listener = new OneToOneResilientIndexingListener(dbClient, dbSession, itemsByIssueKey.values()); BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, listener); bulkIndexer.start(); try (IssueIterator issues = issueIteratorFactory.createForIssueKeys(itemsByIssueKey.keySet())) { while (issues.hasNext()) { IssueDoc issue = issues.next(); bulkIndexer.add(newIndexRequest(issue)); itemsByIssueKey.removeAll(issue.getId()); } } // the remaining uuids reference issues that don't exist in db. They must // be deleted from index. itemsByIssueKey.values().forEach( item -> bulkIndexer.addDeletion(INDEX_TYPE_ISSUE, item.getDocId(), item.getDocRouting())); return bulkIndexer.stop(); }