How to use

ElasticSearchIndexer

in

org.apache.james.backends.es

private void handleEvent(User user, QuotaUsageUpdatedEvent event) throws JsonProcessingException {
  indexer.index(user.getUserName(), 
      quotaRatioToElasticSearchJson.convertToJson(user.getUserName(), event));
}

@Override
public void delete(MailboxSession session, Mailbox mailbox, List<MessageUid> expungedUids) throws MailboxException {
  try {
    elasticSearchIndexer.delete(expungedUids.stream()
      .map(uid ->  indexIdFor(mailbox, uid))
      .collect(Collectors.toList()));
  } catch (Exception e) {
    if (LOGGER.isErrorEnabled()) {
      LOGGER.error("Error when deleting messages {} in mailbox {} from index", mailbox.getMailboxId().serialize(), expungedUids.toArray(), e);
    }
  }
}

@Override
public void deleteAll(MailboxSession session, Mailbox mailbox) throws MailboxException {
  try {
    elasticSearchIndexer.deleteAllMatchingQuery(
      termQuery(
        JsonMessageConstants.MAILBOX_ID,
        mailbox.getMailboxId().serialize()));
  } catch (Exception e) {
    LOGGER.error("Error when deleting all messages in mailbox {}", mailbox.getMailboxId().serialize(), e);
  }
}

@Override
public void update(MailboxSession session, Mailbox mailbox, List<UpdatedFlags> updatedFlagsList) throws MailboxException {
  try {
    elasticSearchIndexer.update(updatedFlagsList.stream()
      .map(updatedFlags -> createUpdatedDocumentPartFromUpdatedFlags(mailbox, updatedFlags))
      .collect(Collectors.toList()));
  } catch (Exception e) {
    LOGGER.error("Error when updating index on mailbox {}", mailbox.getMailboxId().serialize(), e);
  }
}

@Before
public void setUp() {
  client = QuotaSearchIndexCreationUtil.prepareDefaultClient(
    new TestingClientProvider(embeddedElasticSearch.getNode()).get());
  quotaMailboxListener = new ElasticSearchQuotaMailboxListener(
    new ElasticSearchIndexer(client,
      Executors.newSingleThreadExecutor(),
      QuotaRatioElasticSearchConstants.DEFAULT_QUOTA_RATIO_WRITE_ALIAS,
      QuotaRatioElasticSearchConstants.QUOTA_RATIO_TYPE,
      BATCH_SIZE),
    new QuotaRatioToElasticSearchJson());
}

public IndexResponse index(String id, String content) {
  checkArgument(content);
  if (LOGGER.isDebugEnabled()) {
    LOGGER.debug("Indexing {}: {}", id, StringUtils.left(content, DEBUG_MAX_LENGTH_CONTENT));
  }
  return client.prepareIndex(aliasName.getValue(), typeName.getValue(), id)
    .setSource(content)
    .get();
}

@Test
public void updateShouldWork() throws Exception {
  //Given
  Mailbox mailbox = mock(Mailbox.class);
  Flags flags = new Flags();
  UpdatedFlags updatedFlags = UpdatedFlags.builder()
    .uid(MESSAGE_UID)
    .modSeq(MODSEQ)
    .oldFlags(flags)
    .newFlags(flags)
    .build();
  when(mailbox.getMailboxId())
    .thenReturn(MAILBOX_ID);
  when(messageToElasticSearchJson.getUpdatedJsonMessagePart(any(Flags.class), any(Long.class)))
    .thenReturn("json updated content");
  
  //When
  testee.update(session, mailbox, Lists.newArrayList(updatedFlags));
  
  //Then
  ImmutableList<UpdatedRepresentation> expectedUpdatedRepresentations = ImmutableList.of(new UpdatedRepresentation(ELASTIC_SEARCH_ID, "json updated content"));
  verify(elasticSearchIndexer).update(expectedUpdatedRepresentations);
}

new ElasticSearchIndexer(client, Executors.newSingleThreadExecutor(),
  QuotaRatioElasticSearchConstants.DEFAULT_QUOTA_RATIO_WRITE_ALIAS,
  QuotaRatioElasticSearchConstants.QUOTA_RATIO_TYPE),

@Test
public void addShouldIndexEmailBodyWhenNotIndexableAttachment() throws Exception {
  //Given
  Mailbox mailbox = mock(Mailbox.class);
  when(mailbox.getMailboxId())
    .thenReturn(MAILBOX_ID);
  
  MailboxMessage message = mockedMessage(MESSAGE_UID);
  
  when(messageToElasticSearchJson.convertToJson(eq(message), eq(users)))
    .thenThrow(JsonProcessingException.class);
  
  when(messageToElasticSearchJson.convertToJsonWithoutAttachment(eq(message), eq(users)))
    .thenReturn(EXPECTED_JSON_CONTENT);
  
  //When
  testee.add(session, mailbox, message);
  
  //Then
  verify(elasticSearchIndexer).index(eq(ELASTIC_SEARCH_ID), eq(EXPECTED_JSON_CONTENT));
}

@Test
@SuppressWarnings("unchecked")
public void deleteShouldWork() throws Exception {
  //Given
  Mailbox mailbox = mock(Mailbox.class);
  when(mailbox.getMailboxId())
    .thenReturn(MAILBOX_ID);
  BulkResponse expectedBulkResponse = mock(BulkResponse.class);
  when(elasticSearchIndexer.delete(any(List.class)))
    .thenReturn(Optional.of(expectedBulkResponse));
  //When
  testee.delete(session, mailbox, Lists.newArrayList(MESSAGE_UID));
  //Then
  verify(elasticSearchIndexer).delete(eq(Lists.newArrayList(ELASTIC_SEARCH_ID)));
}

@Test
public void updateShouldNotPropagateExceptionWhenExceptionOccurs() throws Exception {
  //Given
  Mailbox mailbox = mock(Mailbox.class);
  Flags flags = new Flags();
  UpdatedFlags updatedFlags = UpdatedFlags.builder()
    .uid(MESSAGE_UID)
    .modSeq(MODSEQ)
    .oldFlags(flags)
    .newFlags(flags)
    .build();
  when(mailbox.getMailboxId())
    .thenReturn(MAILBOX_ID);
  ImmutableList<UpdatedRepresentation> expectedUpdatedRepresentations = ImmutableList.of(new UpdatedRepresentation(ELASTIC_SEARCH_ID, "json updated content"));
  when(elasticSearchIndexer.update(expectedUpdatedRepresentations))
    .thenThrow(new ElasticsearchException(""));
  
  //When
  testee.update(session, mailbox, Lists.newArrayList(updatedFlags));
  
  //Then
  //No exception
}

@Override
protected void initializeMailboxManager() throws Exception {
  Client client = MailboxIndexCreationUtil.prepareDefaultClient(
    new TestingClientProvider(embeddedElasticSearch.getNode()).get());
  storeMailboxManager = new InMemoryIntegrationResources()
    .createMailboxManager(new SimpleGroupMembershipResolver());
  ElasticSearchListeningMessageSearchIndex elasticSearchListeningMessageSearchIndex = new ElasticSearchListeningMessageSearchIndex(
    storeMailboxManager.getMapperFactory(),
    new ElasticSearchIndexer(client,
      Executors.newSingleThreadExecutor(),
      MailboxElasticSearchConstants.DEFAULT_MAILBOX_WRITE_ALIAS,
      MailboxElasticSearchConstants.MESSAGE_TYPE,
      BATCH_SIZE),
    new ElasticSearchSearcher(client, new QueryConverter(new CriterionConverter()), SEARCH_SIZE,
      new InMemoryId.Factory(), storeMailboxManager.getMessageIdFactory(),
      MailboxElasticSearchConstants.DEFAULT_MAILBOX_READ_ALIAS,
      MailboxElasticSearchConstants.MESSAGE_TYPE),
    new MessageToElasticSearchJson(textExtractor, ZoneId.of("Europe/Paris"), IndexAttachments.YES));
  messageIdManager = new StoreMessageIdManager(
    storeMailboxManager,
    storeMailboxManager.getMapperFactory(),
    storeMailboxManager.getEventDispatcher(),
    storeMailboxManager.getMessageIdFactory(),
    storeMailboxManager.getQuotaManager(),
    storeMailboxManager.getQuotaRootResolver());
  storeMailboxManager.setMessageSearchIndex(elasticSearchListeningMessageSearchIndex);
  storeMailboxManager.addGlobalListener(elasticSearchListeningMessageSearchIndex, new MockMailboxSession("admin"));
  this.messageSearchIndex = elasticSearchListeningMessageSearchIndex;
}

@Test
public void deleteAllShouldNotPropagateExceptionWhenExceptionOccurs() throws Exception {
  //Given
  Mailbox mailbox = mock(Mailbox.class);
  when(mailbox.getMailboxId())
    .thenReturn(MAILBOX_ID);
  doThrow(RuntimeException.class)
    .when(elasticSearchIndexer).deleteAllMatchingQuery(QueryBuilders.termQuery("mailboxId", "12"));
  //When
  testee.deleteAll(session, mailbox);
  
  //Then
  //No Exception
}

@Test
public void addShouldIndex() throws Exception {
  //Given
  Mailbox mailbox = mock(Mailbox.class);
  when(mailbox.getMailboxId())
    .thenReturn(MAILBOX_ID);
  MailboxMessage message = mockedMessage(MESSAGE_UID);
  
  when(messageToElasticSearchJson.convertToJson(eq(message), eq(users)))
    .thenReturn(EXPECTED_JSON_CONTENT);
  
  //When
  testee.add(session, mailbox, message);
  
  //Then
  verify(elasticSearchIndexer).index(eq(ELASTIC_SEARCH_ID), eq(EXPECTED_JSON_CONTENT));
}

@Test
@SuppressWarnings("unchecked")
public void deleteShouldNotPropagateExceptionWhenExceptionOccurs() throws Exception {
  //Given
  Mailbox mailbox = mock(Mailbox.class);
  when(mailbox.getMailboxId())
    .thenReturn(MAILBOX_ID);
  
  when(elasticSearchIndexer.delete(any(List.class)))
    .thenThrow(new ElasticsearchException(""));
  
  //When
  testee.delete(session, mailbox, Lists.newArrayList(MESSAGE_UID));
  
  //Then
  //No exception
}

@Test
public void deleteAllShouldWork() throws Exception {
  //Given
  Mailbox mailbox = mock(Mailbox.class);
  when(mailbox.getMailboxId())
    .thenReturn(MAILBOX_ID);
  //When
  testee.deleteAll(session, mailbox);
  
  //Then
  QueryBuilder expectedQueryBuilder = QueryBuilders.termQuery("mailboxId", "12");
  verify(elasticSearchIndexer).deleteAllMatchingQuery(refEq(expectedQueryBuilder));
}

@Override
public void add(MailboxSession session, Mailbox mailbox, MailboxMessage message) throws MailboxException {
  try {
    LOGGER.info("Indexing mailbox {}-{} of user {} on message {}",
        mailbox.getName(),
        mailbox.getMailboxId(),
        session.getUser().getUserName(),
        message.getUid());
    elasticSearchIndexer.index(indexIdFor(mailbox, message.getUid()), messageToElasticSearchJson.convertToJson(message, ImmutableList.of(session.getUser())));
  } catch (Exception e) {
    try {
      LOGGER.warn("Indexing mailbox {}-{} of user {} on message {} without attachments ",
          mailbox.getName(),
          mailbox.getMailboxId().serialize(),
          session.getUser().getUserName(),
          message.getUid(),
          e);
      elasticSearchIndexer.index(indexIdFor(mailbox, message.getUid()), messageToElasticSearchJson.convertToJsonWithoutAttachment(message, ImmutableList.of(session.getUser())));
    } catch (JsonProcessingException e1) {
      LOGGER.error("Error when indexing mailbox {}-{} of user {} on message {} without its attachment",
          mailbox.getName(),
          mailbox.getMailboxId().serialize(),
          session.getUser().getUserName(),
          message.getUid(),
          e1);
    }
  }
}

@Test
@SuppressWarnings("unchecked")
public void deleteShouldWorkWhenMultipleMessageIds() throws Exception {
  //Given
  Mailbox mailbox = mock(Mailbox.class);
  MessageUid messageId2 = MessageUid.of(2);
  MessageUid messageId3 = MessageUid.of(3);
  MessageUid messageId4 = MessageUid.of(4);
  MessageUid messageId5 = MessageUid.of(5);
  when(mailbox.getMailboxId())
    .thenReturn(MAILBOX_ID);
  BulkResponse expectedBulkResponse = mock(BulkResponse.class);
  when(elasticSearchIndexer.delete(any(List.class)))
    .thenReturn(Optional.of(expectedBulkResponse));
  
  //When
  testee.delete(session, mailbox, Lists.newArrayList(MESSAGE_UID, messageId2, messageId3, messageId4, messageId5));
  
  //Then
  verify(elasticSearchIndexer).delete(eq(Lists.newArrayList(ELASTIC_SEARCH_ID, "12:2", "12:3", "12:4", "12:5")));
}