@Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { IndexingResult result = new IndexingResult(); if (!items.isEmpty()) { ListMultimap<IndexType, EsQueueDto> itemsByType = groupItemsByType(items); result.add(doIndexRules(dbSession, itemsByType.get(INDEX_TYPE_RULE))); result.add(doIndexRuleExtensions(dbSession, itemsByType.get(INDEX_TYPE_RULE_EXTENSION))); } return result; }
@Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { ListMultimap<String, EsQueueDto> itemsByIssueKey = ArrayListMultimap.create(); ListMultimap<String, EsQueueDto> itemsByProjectKey = ArrayListMultimap.create(); items.forEach(i -> { if (ID_TYPE_ISSUE_KEY.equals(i.getDocIdType())) { itemsByIssueKey.put(i.getDocId(), i); } else if (ID_TYPE_PROJECT_UUID.equals(i.getDocIdType())) { itemsByProjectKey.put(i.getDocId(), i); } else { LOGGER.error("Unsupported es_queue.doc_id_type for issues. Manual fix is required: " + i); } }); IndexingResult result = new IndexingResult(); result.add(doIndexIssueItems(dbSession, itemsByIssueKey)); result.add(doIndexProjectItems(dbSession, itemsByProjectKey)); return result; }
/** * @return the number of items that have been successfully indexed */ @Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { IndexingResult result = new IndexingResult(); if (items.isEmpty()) { return result; } Map<Long, EsQueueDto> activeRuleItems = new HashMap<>(); Map<String, EsQueueDto> ruleProfileItems = new HashMap<>(); items.forEach(i -> { if (ID_TYPE_RULE_PROFILE_UUID.equals(i.getDocIdType())) { ruleProfileItems.put(i.getDocId(), i); } else if (ID_TYPE_ACTIVE_RULE_ID.equals(i.getDocIdType())) { activeRuleItems.put(Long.parseLong(i.getDocId()), i); } else { LOGGER.error("Unsupported es_queue.doc_id_type. Removing row from queue: " + i); deleteQueueDto(dbSession, i); } }); if (!activeRuleItems.isEmpty()) { result.add(doIndexActiveRules(dbSession, activeRuleItems)); } if (!ruleProfileItems.isEmpty()) { result.add(doIndexRuleProfiles(dbSession, ruleProfileItems)); } return result; }
@Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { IndexingResult result = new IndexingResult(); List<BulkIndexer> bulkIndexers = items.stream() .map(EsQueueDto::getDocType) .distinct() .map(IndexType::parse) .filter(indexTypes::contains) .map(indexType -> new BulkIndexer(esClient, indexType, Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items))) .collect(Collectors.toList()); if (bulkIndexers.isEmpty()) { return result; } bulkIndexers.forEach(BulkIndexer::start); PermissionIndexerDao permissionIndexerDao = new PermissionIndexerDao(); Set<String> remainingProjectUuids = items.stream().map(EsQueueDto::getDocId).collect(MoreCollectors.toHashSet()); permissionIndexerDao.selectByUuids(dbClient, dbSession, remainingProjectUuids).forEach(p -> { remainingProjectUuids.remove(p.getProjectUuid()); bulkIndexers.forEach(bi -> bi.add(newIndexRequest(p, bi.getIndexType()))); }); // the remaining references on projects that don't exist in db. They must // be deleted from index. remainingProjectUuids.forEach(projectUuid -> bulkIndexers.forEach(bi -> bi.addDeletion(bi.getIndexType(), projectUuid, projectUuid))); bulkIndexers.forEach(b -> result.add(b.stop())); return result; }
@VisibleForTesting void recover() { try (DbSession dbSession = dbClient.openSession(false)) { Profiler profiler = Profiler.create(LOGGER).start(); long beforeDate = system2.now() - minAgeInMs; IndexingResult result = new IndexingResult(); Collection<EsQueueDto> items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit); while (!items.isEmpty()) { IndexingResult loopResult = new IndexingResult(); groupItemsByType(items).asMap().forEach((type, typeItems) -> loopResult.add(doIndex(dbSession, type, typeItems))); result.add(loopResult); if (loopResult.getSuccessRatio() <= CIRCUIT_BREAKER_IN_PERCENT) { LOGGER.error(LOG_PREFIX + "too many failures [{}/{} documents], waiting for next run", loopResult.getFailures(), loopResult.getTotal()); break; } if (loopResult.getTotal() == 0L) { break; } items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit); } if (result.getTotal() > 0L) { profiler.stopInfo(LOG_PREFIX + format("%d documents processed [%d failures]", result.getTotal(), result.getFailures())); } } catch (Throwable t) { LOGGER.error(LOG_PREFIX + "fail to recover documents", t); } }
private IndexingResult doIndexRuleProfiles(DbSession dbSession, Map<String, EsQueueDto> ruleProfileItems) { IndexingResult result = new IndexingResult(); for (Map.Entry<String, EsQueueDto> entry : ruleProfileItems.entrySet()) { String ruleProfileUUid = entry.getKey(); EsQueueDto item = entry.getValue(); IndexingResult profileResult; RulesProfileDto profile = dbClient.qualityProfileDao().selectRuleProfile(dbSession, ruleProfileUUid); if (profile == null) { // profile does not exist anymore in db --> related documents must be deleted from index rules/activeRule SearchRequestBuilder search = esClient.prepareSearch(INDEX_TYPE_ACTIVE_RULE) .setQuery(QueryBuilders.boolQuery().must(termQuery(FIELD_ACTIVE_RULE_PROFILE_UUID, ruleProfileUUid))); profileResult = BulkIndexer.delete(esClient, INDEX_TYPE_ACTIVE_RULE, search); } else { BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, IndexingListener.FAIL_ON_ERROR); bulkIndexer.start(); dbClient.activeRuleDao().scrollByRuleProfileForIndexing(dbSession, ruleProfileUUid, i -> bulkIndexer.add(newIndexRequest(i))); profileResult = bulkIndexer.stop(); } if (profileResult.isSuccess()) { deleteQueueDto(dbSession, item); } result.add(profileResult); } return result; }
@Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { IndexingResult result = new IndexingResult(); if (!items.isEmpty()) { ListMultimap<IndexType, EsQueueDto> itemsByType = groupItemsByType(items); result.add(doIndexRules(dbSession, itemsByType.get(INDEX_TYPE_RULE))); result.add(doIndexRuleExtensions(dbSession, itemsByType.get(INDEX_TYPE_RULE_EXTENSION))); } return result; }
@Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { ListMultimap<String, EsQueueDto> itemsByIssueKey = ArrayListMultimap.create(); ListMultimap<String, EsQueueDto> itemsByProjectKey = ArrayListMultimap.create(); items.forEach(i -> { if (ID_TYPE_ISSUE_KEY.equals(i.getDocIdType())) { itemsByIssueKey.put(i.getDocId(), i); } else if (ID_TYPE_PROJECT_UUID.equals(i.getDocIdType())) { itemsByProjectKey.put(i.getDocId(), i); } else { LOGGER.error("Unsupported es_queue.doc_id_type for issues. Manual fix is required: " + i); } }); IndexingResult result = new IndexingResult(); result.add(doIndexIssueItems(dbSession, itemsByIssueKey)); result.add(doIndexProjectItems(dbSession, itemsByProjectKey)); return result; }
/** * @return the number of items that have been successfully indexed */ @Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { IndexingResult result = new IndexingResult(); if (items.isEmpty()) { return result; } Map<Long, EsQueueDto> activeRuleItems = new HashMap<>(); Map<String, EsQueueDto> ruleProfileItems = new HashMap<>(); items.forEach(i -> { if (ID_TYPE_RULE_PROFILE_UUID.equals(i.getDocIdType())) { ruleProfileItems.put(i.getDocId(), i); } else if (ID_TYPE_ACTIVE_RULE_ID.equals(i.getDocIdType())) { activeRuleItems.put(Long.parseLong(i.getDocId()), i); } else { LOGGER.error("Unsupported es_queue.doc_id_type. Removing row from queue: " + i); deleteQueueDto(dbSession, i); } }); if (!activeRuleItems.isEmpty()) { result.add(doIndexActiveRules(dbSession, activeRuleItems)); } if (!ruleProfileItems.isEmpty()) { result.add(doIndexRuleProfiles(dbSession, ruleProfileItems)); } return result; }
@Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { IndexingResult result = new IndexingResult(); List<BulkIndexer> bulkIndexers = items.stream() .map(EsQueueDto::getDocType) .distinct() .map(IndexType::parse) .filter(indexTypes::contains) .map(indexType -> new BulkIndexer(esClient, indexType, Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items))) .collect(Collectors.toList()); if (bulkIndexers.isEmpty()) { return result; } bulkIndexers.forEach(BulkIndexer::start); PermissionIndexerDao permissionIndexerDao = new PermissionIndexerDao(); Set<String> remainingProjectUuids = items.stream().map(EsQueueDto::getDocId).collect(MoreCollectors.toHashSet()); permissionIndexerDao.selectByUuids(dbClient, dbSession, remainingProjectUuids).forEach(p -> { remainingProjectUuids.remove(p.getProjectUuid()); bulkIndexers.forEach(bi -> bi.add(newIndexRequest(p, bi.getIndexType()))); }); // the remaining references on projects that don't exist in db. They must // be deleted from index. remainingProjectUuids.forEach(projectUuid -> bulkIndexers.forEach(bi -> bi.addDeletion(bi.getIndexType(), projectUuid, projectUuid))); bulkIndexers.forEach(b -> result.add(b.stop())); return result; }
@VisibleForTesting void recover() { try (DbSession dbSession = dbClient.openSession(false)) { Profiler profiler = Profiler.create(LOGGER).start(); long beforeDate = system2.now() - minAgeInMs; IndexingResult result = new IndexingResult(); Collection<EsQueueDto> items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit); while (!items.isEmpty()) { IndexingResult loopResult = new IndexingResult(); groupItemsByType(items).asMap().forEach((type, typeItems) -> loopResult.add(doIndex(dbSession, type, typeItems))); result.add(loopResult); if (loopResult.getSuccessRatio() <= CIRCUIT_BREAKER_IN_PERCENT) { LOGGER.error(LOG_PREFIX + "too many failures [{}/{} documents], waiting for next run", loopResult.getFailures(), loopResult.getTotal()); break; } if (loopResult.getTotal() == 0L) { break; } items = dbClient.esQueueDao().selectForRecovery(dbSession, beforeDate, loopLimit); } if (result.getTotal() > 0L) { profiler.stopInfo(LOG_PREFIX + format("%d documents processed [%d failures]", result.getTotal(), result.getFailures())); } } catch (Throwable t) { LOGGER.error(LOG_PREFIX + "fail to recover documents", t); } }
private IndexingResult doIndexRuleProfiles(DbSession dbSession, Map<String, EsQueueDto> ruleProfileItems) { IndexingResult result = new IndexingResult(); for (Map.Entry<String, EsQueueDto> entry : ruleProfileItems.entrySet()) { String ruleProfileUUid = entry.getKey(); EsQueueDto item = entry.getValue(); IndexingResult profileResult; RulesProfileDto profile = dbClient.qualityProfileDao().selectRuleProfile(dbSession, ruleProfileUUid); if (profile == null) { // profile does not exist anymore in db --> related documents must be deleted from index rules/activeRule SearchRequestBuilder search = esClient.prepareSearch(INDEX_TYPE_ACTIVE_RULE) .setQuery(QueryBuilders.boolQuery().must(termQuery(FIELD_ACTIVE_RULE_PROFILE_UUID, ruleProfileUUid))); profileResult = BulkIndexer.delete(esClient, INDEX_TYPE_ACTIVE_RULE, search); } else { BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, IndexingListener.FAIL_ON_ERROR); bulkIndexer.start(); dbClient.activeRuleDao().scrollByRuleProfileForIndexing(dbSession, ruleProfileUUid, i -> bulkIndexer.add(newIndexRequest(i))); profileResult = bulkIndexer.stop(); } if (profileResult.isSuccess()) { deleteQueueDto(dbSession, item); } result.add(profileResult); } return result; }