private static RuleExtensionId explodeRuleExtensionDocId(EsQueueDto esQueueDto) { checkArgument(Objects.equals(esQueueDto.getDocType(), "rules/ruleExtension")); return new RuleExtensionId(esQueueDto.getDocId()); }
@Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { ListMultimap<String, EsQueueDto> itemsByIssueKey = ArrayListMultimap.create(); ListMultimap<String, EsQueueDto> itemsByProjectKey = ArrayListMultimap.create(); items.forEach(i -> { if (ID_TYPE_ISSUE_KEY.equals(i.getDocIdType())) { itemsByIssueKey.put(i.getDocId(), i); } else if (ID_TYPE_PROJECT_UUID.equals(i.getDocIdType())) { itemsByProjectKey.put(i.getDocId(), i); } else { LOGGER.error("Unsupported es_queue.doc_id_type for issues. Manual fix is required: " + i); } }); IndexingResult result = new IndexingResult(); result.add(doIndexIssueItems(dbSession, itemsByIssueKey)); result.add(doIndexProjectItems(dbSession, itemsByProjectKey)); return result; }
public OneToOneResilientIndexingListener(DbClient dbClient, DbSession dbSession, Collection<EsQueueDto> items) { this.dbClient = dbClient; this.dbSession = dbSession; this.itemsById = items.stream() .collect(MoreCollectors.index(i -> new DocId(IndexType.parse(i.getDocType()), i.getDocId()), Function.identity())); }
/** * @return the number of items that have been successfully indexed */ @Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { IndexingResult result = new IndexingResult(); if (items.isEmpty()) { return result; } Map<Long, EsQueueDto> activeRuleItems = new HashMap<>(); Map<String, EsQueueDto> ruleProfileItems = new HashMap<>(); items.forEach(i -> { if (ID_TYPE_RULE_PROFILE_UUID.equals(i.getDocIdType())) { ruleProfileItems.put(i.getDocId(), i); } else if (ID_TYPE_ACTIVE_RULE_ID.equals(i.getDocIdType())) { activeRuleItems.put(Long.parseLong(i.getDocId()), i); } else { LOGGER.error("Unsupported es_queue.doc_id_type. Removing row from queue: " + i); deleteQueueDto(dbSession, i); } }); if (!activeRuleItems.isEmpty()) { result.add(doIndexActiveRules(dbSession, activeRuleItems)); } if (!ruleProfileItems.isEmpty()) { result.add(doIndexRuleProfiles(dbSession, ruleProfileItems)); } return result; }
/** * Commits the DB transaction and adds the issues to Elasticsearch index. * <p> * If indexing fails, then the recovery daemon will retry later and this * method successfully returns. Meanwhile these issues will be "eventually * consistent" when requesting the index. */ public void commitAndIndexIssues(DbSession dbSession, Collection<IssueDto> issues) { ListMultimap<String, EsQueueDto> itemsByIssueKey = ArrayListMultimap.create(); issues.stream() .map(issue -> createQueueDto(issue.getKey(), ID_TYPE_ISSUE_KEY, issue.getProjectUuid())) // a mutable ListMultimap is needed for doIndexIssueItems, so MoreCollectors.index() is // not used .forEach(i -> itemsByIssueKey.put(i.getDocId(), i)); dbClient.esQueueDao().insert(dbSession, itemsByIssueKey.values()); dbSession.commit(); doIndexIssueItems(dbSession, itemsByIssueKey); }
@Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { ListMultimap<String, EsQueueDto> itemsByIssueKey = ArrayListMultimap.create(); ListMultimap<String, EsQueueDto> itemsByProjectKey = ArrayListMultimap.create(); items.forEach(i -> { if (ID_TYPE_ISSUE_KEY.equals(i.getDocIdType())) { itemsByIssueKey.put(i.getDocId(), i); } else if (ID_TYPE_PROJECT_UUID.equals(i.getDocIdType())) { itemsByProjectKey.put(i.getDocId(), i); } else { LOGGER.error("Unsupported es_queue.doc_id_type for issues. Manual fix is required: " + i); } }); IndexingResult result = new IndexingResult(); result.add(doIndexIssueItems(dbSession, itemsByIssueKey)); result.add(doIndexProjectItems(dbSession, itemsByProjectKey)); return result; }
private IndexingResult doIndexActiveRules(DbSession dbSession, Map<Long, EsQueueDto> activeRuleItems) { OneToOneResilientIndexingListener listener = new OneToOneResilientIndexingListener(dbClient, dbSession, activeRuleItems.values()); BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, listener); bulkIndexer.start(); Map<Long, EsQueueDto> remaining = new HashMap<>(activeRuleItems); dbClient.activeRuleDao().scrollByIdsForIndexing(dbSession, activeRuleItems.keySet(), i -> { remaining.remove(i.getId()); bulkIndexer.add(newIndexRequest(i)); }); // the remaining ids reference rows that don't exist in db. They must // be deleted from index. remaining.values().forEach(item -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE, item.getDocId(), item.getDocRouting())); return bulkIndexer.stop(); }
/** * @return the number of items that have been successfully indexed */ @Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { IndexingResult result = new IndexingResult(); if (items.isEmpty()) { return result; } Map<Long, EsQueueDto> activeRuleItems = new HashMap<>(); Map<String, EsQueueDto> ruleProfileItems = new HashMap<>(); items.forEach(i -> { if (ID_TYPE_RULE_PROFILE_UUID.equals(i.getDocIdType())) { ruleProfileItems.put(i.getDocId(), i); } else if (ID_TYPE_ACTIVE_RULE_ID.equals(i.getDocIdType())) { activeRuleItems.put(Long.parseLong(i.getDocId()), i); } else { LOGGER.error("Unsupported es_queue.doc_id_type. Removing row from queue: " + i); deleteQueueDto(dbSession, i); } }); if (!activeRuleItems.isEmpty()) { result.add(doIndexActiveRules(dbSession, activeRuleItems)); } if (!ruleProfileItems.isEmpty()) { result.add(doIndexRuleProfiles(dbSession, ruleProfileItems)); } return result; }
private IndexingResult doIndexRules(DbSession dbSession, List<EsQueueDto> items) { BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items)); bulkIndexer.start(); Set<Integer> ruleIds = items .stream() .map(i -> Integer.parseInt(i.getDocId())) .collect(toHashSet(items.size())); dbClient.ruleDao().scrollIndexingRulesByKeys(dbSession, ruleIds, r -> { bulkIndexer.add(newRuleDocIndexRequest(r)); bulkIndexer.add(newRuleExtensionDocIndexRequest(r)); ruleIds.remove(r.getId()); }); // the remaining items reference rows that don't exist in db. They must // be deleted from index. ruleIds.forEach(ruleId -> { bulkIndexer.addDeletion(INDEX_TYPE_RULE, ruleId.toString(), ruleId.toString()); bulkIndexer.addDeletion(INDEX_TYPE_RULE_EXTENSION, RuleExtensionDoc.idOf(ruleId, RuleExtensionScope.system()), ruleId.toString()); }); return bulkIndexer.stop(); }
private static DocId toDocId(EsQueueDto dto) { return new DocId(IndexType.parse(dto.getDocType()), dto.getDocId()); }
@Test public void recover_multiple_times_the_same_document() { EsQueueDto item1 = insertItem(FOO_TYPE, "f1"); EsQueueDto item2 = insertItem(FOO_TYPE, item1.getDocId()); EsQueueDto item3 = insertItem(FOO_TYPE, item1.getDocId()); advanceInTime(); SuccessfulFakeIndexer indexer = new SuccessfulFakeIndexer(FOO_TYPE); underTest = newRecoveryIndexer(indexer); underTest.recover(); assertThatQueueHasSize(0); assertThat(indexer.called).hasSize(1); assertThat(indexer.called.get(0)).extracting(EsQueueDto::getUuid) .containsExactlyInAnyOrder(item1.getUuid(), item2.getUuid(), item3.getUuid()); assertThatLogsContain(TRACE, "Elasticsearch recovery - processing 3 [foos/foo]"); assertThatLogsContain(INFO, "Elasticsearch recovery - 3 documents processed [0 failures]"); }
/** * ES_QUEUE can contain multiple times the same document, for instance * when an issue has been updated multiple times in a row without * being successfully indexed. * Elasticsearch response does not make difference between the different * occurrences (and nevertheless it would be useless). So all the * occurrences are marked as successfully indexed if a single request * passes. */ @Test public void onSuccess_deletes_all_the_rows_with_same_doc_id() { EsQueueDto item1 = insertInQueue(INDEX_TYPE_ISSUE, "foo"); // same id as item1 EsQueueDto item2 = insertInQueue(INDEX_TYPE_ISSUE, item1.getDocId()); EsQueueDto item3 = insertInQueue(INDEX_TYPE_ISSUE, "bar"); db.commit(); IndexingListener underTest = newListener(asList(item1, item2, item3)); underTest.onSuccess(asList(toDocId(item1))); assertThatEsTableContainsOnly(item3); }
private IndexingResult doIndexIssueItems(DbSession dbSession, ListMultimap<String, EsQueueDto> itemsByIssueKey) { if (itemsByIssueKey.isEmpty()) { return new IndexingResult(); } IndexingListener listener = new OneToOneResilientIndexingListener(dbClient, dbSession, itemsByIssueKey.values()); BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, listener); bulkIndexer.start(); try (IssueIterator issues = issueIteratorFactory.createForIssueKeys(itemsByIssueKey.keySet())) { while (issues.hasNext()) { IssueDoc issue = issues.next(); bulkIndexer.add(newIndexRequest(issue)); itemsByIssueKey.removeAll(issue.getId()); } } // the remaining uuids reference issues that don't exist in db. They must // be deleted from index. itemsByIssueKey.values().forEach( item -> bulkIndexer.addDeletion(INDEX_TYPE_ISSUE, item.getDocId(), item.getDocRouting())); return bulkIndexer.stop(); }
private static RuleExtensionId explodeRuleExtensionDocId(EsQueueDto esQueueDto) { checkArgument(Objects.equals(esQueueDto.getDocType(), "rules/ruleExtension")); return new RuleExtensionId(esQueueDto.getDocId()); }
public OneToOneResilientIndexingListener(DbClient dbClient, DbSession dbSession, Collection<EsQueueDto> items) { this.dbClient = dbClient; this.dbSession = dbSession; this.itemsById = items.stream() .collect(MoreCollectors.index(i -> new DocId(IndexType.parse(i.getDocType()), i.getDocId()), Function.identity())); }
/** * Commits the DB transaction and adds the issues to Elasticsearch index. * <p> * If indexing fails, then the recovery daemon will retry later and this * method successfully returns. Meanwhile these issues will be "eventually * consistent" when requesting the index. */ public void commitAndIndexIssues(DbSession dbSession, Collection<IssueDto> issues) { ListMultimap<String, EsQueueDto> itemsByIssueKey = ArrayListMultimap.create(); issues.stream() .map(issue -> createQueueDto(issue.getKey(), ID_TYPE_ISSUE_KEY, issue.getProjectUuid())) // a mutable ListMultimap is needed for doIndexIssueItems, so MoreCollectors.index() is // not used .forEach(i -> itemsByIssueKey.put(i.getDocId(), i)); dbClient.esQueueDao().insert(dbSession, itemsByIssueKey.values()); dbSession.commit(); doIndexIssueItems(dbSession, itemsByIssueKey); }
@Override public IndexingResult index(DbSession dbSession, Collection<EsQueueDto> items) { // The items are to be deleted if (items.isEmpty()) { return new IndexingResult(); } IndexingListener listener = new OneToManyResilientIndexingListener(dbClient, dbSession, items); BulkIndexer bulkIndexer = new BulkIndexer(esClient, INDEX_TYPE_TEST, Size.REGULAR, listener); bulkIndexer.start(); items.forEach(i -> { String projectUuid = i.getDocId(); addProjectDeletionToBulkIndexer(bulkIndexer, projectUuid); }); return bulkIndexer.stop(); }
private IndexingResult doIndexActiveRules(DbSession dbSession, Map<Long, EsQueueDto> activeRuleItems) { OneToOneResilientIndexingListener listener = new OneToOneResilientIndexingListener(dbClient, dbSession, activeRuleItems.values()); BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, listener); bulkIndexer.start(); Map<Long, EsQueueDto> remaining = new HashMap<>(activeRuleItems); dbClient.activeRuleDao().scrollByIdsForIndexing(dbSession, activeRuleItems.keySet(), i -> { remaining.remove(i.getId()); bulkIndexer.add(newIndexRequest(i)); }); // the remaining ids reference rows that don't exist in db. They must // be deleted from index. remaining.values().forEach(item -> bulkIndexer.addDeletion(RuleIndexDefinition.INDEX_TYPE_ACTIVE_RULE, item.getDocId(), item.getDocRouting())); return bulkIndexer.stop(); }
private IndexingResult doIndexRules(DbSession dbSession, List<EsQueueDto> items) { BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, new OneToOneResilientIndexingListener(dbClient, dbSession, items)); bulkIndexer.start(); Set<Integer> ruleIds = items .stream() .map(i -> Integer.parseInt(i.getDocId())) .collect(toHashSet(items.size())); dbClient.ruleDao().scrollIndexingRulesByKeys(dbSession, ruleIds, r -> { bulkIndexer.add(newRuleDocIndexRequest(r)); bulkIndexer.add(newRuleExtensionDocIndexRequest(r)); ruleIds.remove(r.getId()); }); // the remaining items reference rows that don't exist in db. They must // be deleted from index. ruleIds.forEach(ruleId -> { bulkIndexer.addDeletion(INDEX_TYPE_RULE, ruleId.toString(), ruleId.toString()); bulkIndexer.addDeletion(INDEX_TYPE_RULE_EXTENSION, RuleExtensionDoc.idOf(ruleId, RuleExtensionScope.system()), ruleId.toString()); }); return bulkIndexer.stop(); }
private IndexingResult doIndexIssueItems(DbSession dbSession, ListMultimap<String, EsQueueDto> itemsByIssueKey) { if (itemsByIssueKey.isEmpty()) { return new IndexingResult(); } IndexingListener listener = new OneToOneResilientIndexingListener(dbClient, dbSession, itemsByIssueKey.values()); BulkIndexer bulkIndexer = createBulkIndexer(Size.REGULAR, listener); bulkIndexer.start(); try (IssueIterator issues = issueIteratorFactory.createForIssueKeys(itemsByIssueKey.keySet())) { while (issues.hasNext()) { IssueDoc issue = issues.next(); bulkIndexer.add(newIndexRequest(issue)); itemsByIssueKey.removeAll(issue.getId()); } } // the remaining uuids reference issues that don't exist in db. They must // be deleted from index. itemsByIssueKey.values().forEach( item -> bulkIndexer.addDeletion(INDEX_TYPE_ISSUE, item.getDocId(), item.getDocRouting())); return bulkIndexer.stop(); }