public List<String> bulkIndex(final List<Map.Entry<IndexSet, Message>> messageList, boolean isSystemTraffic) { if (messageList.isEmpty()) { return Collections.emptyList(); } final Bulk.Builder bulk = new Bulk.Builder(); for (Map.Entry<IndexSet, Message> entry : messageList) { final Message message = entry.getValue(); if (isSystemTraffic) { systemTrafficCounter.inc(message.getSize()); } else { outputByteCounter.inc(message.getSize()); } bulk.addAction(new Index.Builder(message.toElasticSearchObject(invalidTimestampMeter)) .index(entry.getKey().getWriteIndexAlias()) .type(IndexMapping.TYPE_MESSAGE) .id(message.getId()) .build()); } final BulkResult result = runBulkRequest(bulk.build(), messageList.size()); final List<BulkResult.BulkResultItem> failedItems = result.getFailedItems(); if (LOG.isDebugEnabled()) { LOG.debug("Index: Bulk indexed {} messages, took {} ms, failures: {}", result.getItems().size(), result, failedItems.size()); } if (!failedItems.isEmpty()) { return propagateFailure(failedItems, messageList, result.getErrorMessage()); } else { return Collections.emptyList(); } }
source, target, bulkResult.getItems().size(), bulkResult.getJsonObject().path("took").asLong(), hasFailedItems);
@Override public List<FailedItem<T>> getFailedItems() { final List<FailedItem<T>> failedItems = new ArrayList<>(); for (int i = 0; i< rawResult.getItems().size(); i++) { final BulkResult.BulkResultItem bulkResultItem = rawResult.getItems().get(i); if (bulkResultItem.error != null && !bulkResultItem.error.isEmpty()) { final Exception cause = BulkUtils.convertToException(bulkResultItem.error); final T failedObject = documents.get(i); final FailedItem failedItem = new FailedItem(i, failedObject, cause); failedItems.add(failedItem); } } return failedItems; }
@Override public List<FailedItem<T>> getFailedItems() { final List<FailedItem<T>> failedItems = new ArrayList<>(); for (int i = 0; i< rawResult.getItems().size(); i++) { final BulkResult.BulkResultItem bulkResultItem = rawResult.getItems().get(i); if (bulkResultItem.error != null && !bulkResultItem.error.isEmpty()) { final Exception cause = BulkUtils.convertToException(bulkResultItem.error); final T failedObject = documents.get(i); final FailedItem failedItem = new FailedItem(i, failedObject, cause); failedItems.add(failedItem); } } return failedItems; }
private List<BackendRequest<?>> getErroneousItems(List<BackendRequest<?>> actions, BulkResult response) { int i = 0; List<BackendRequest<?>> erroneousItems = new ArrayList<>(); for ( BulkResultItem resultItem : response.getItems() ) { // When getting a 404 for a DELETE, the error is null :(, so checking both if ( resultItem.error != null || resultItem.status >= 400 ) { BackendRequest<?> action = actions.get( i ); if ( !action.getIgnoredErrorStatuses().contains( resultItem.status ) ) { erroneousItems.add( action ); } } i++; } return erroneousItems; }
private String resultToString(JestResult result) { StringBuilder sb = new StringBuilder(); sb.append( "Status: " ).append( result.getResponseCode() ).append( "\n" ); sb.append( "Error message: " ).append( result.getErrorMessage() ).append( "\n\n" ); if ( result instanceof BulkResult ) { for ( BulkResultItem item : ( (BulkResult) result ).getItems() ) { sb.append( "Operation: " ).append( item.operation ).append( "\n" ); sb.append( " Index: " ).append( item.index ).append( "\n" ); sb.append( " Type: " ).append( item.type ).append( "\n" ); sb.append( " Id: " ).append( item.id ).append( "\n" ); sb.append( " Status: " ).append( item.status ).append( "\n" ); sb.append( " Error: " ).append( item.error ).append( "\n" ); } } return sb.toString(); } }
List<BulkResult.BulkResultItem> items = brst.getItems(); for (int idx = 0; idx < items.size(); ++idx) { BulkResult.BulkResultItem item = items.get(idx);
public List<String> bulkIndex(final List<Map.Entry<IndexSet, Message>> messageList, boolean isSystemTraffic) { if (messageList.isEmpty()) { return Collections.emptyList(); } final Bulk.Builder bulk = new Bulk.Builder(); for (Map.Entry<IndexSet, Message> entry : messageList) { final Message message = entry.getValue(); if (isSystemTraffic) { systemTrafficCounter.inc(message.getSize()); } else { outputByteCounter.inc(message.getSize()); } bulk.addAction(new Index.Builder(message.toElasticSearchObject(invalidTimestampMeter)) .index(entry.getKey().getWriteIndexAlias()) .type(IndexMapping.TYPE_MESSAGE) .id(message.getId()) .build()); } final BulkResult result = runBulkRequest(bulk.build(), messageList.size()); final List<BulkResult.BulkResultItem> failedItems = result.getFailedItems(); if (LOG.isDebugEnabled()) { LOG.debug("Index: Bulk indexed {} messages, took {} ms, failures: {}", result.getItems().size(), result, failedItems.size()); } if (!failedItems.isEmpty()) { return propagateFailure(failedItems, messageList, result.getErrorMessage()); } else { return Collections.emptyList(); } }
source, target, bulkResult.getItems().size(), bulkResult.getJsonObject().path("took").asLong(), hasFailedItems);
private void sendEvents(final List<Event> allEvents) { final BulkRequest<Event> request = new BulkRequest<>(jestClient, allEvents, (events) -> new BulkWrapper(new Bulk.Builder().addAction(convertEventsToEsActions(events))), bulkRetryCount); try { request.execute(); } catch (BulkException ex) { final BulkResult result = ex.getBulkResult().getRawResult(); LOG.error("Bulk API action failed. Error response was: {}", result.getErrorMessage()); // Log any unsuccessful completions as errors if (LOG.isDebugEnabled() && result != null) { for (BulkResultItem item : result.getItems()) { if (item.status >= 200 && item.status < 300) { logEsDebug(item.operation, item.index, item.type, "none", item.status, item.error); } else { logEsError(item.operation, item.index, item.type, "none", item.status, item.error); } } } } catch (IOException ex) { LOG.error("Bulk API action failed. An exception occurred: {}", ex.getMessage(), ex); } }
private void sendEvents(final List<Event> allEvents) { final BulkRequest<Event> request = new BulkRequest<>(jestClient, allEvents, (events) -> new BulkWrapper(new Bulk.Builder().addAction(convertEventsToEsActions(events))), bulkRetryCount); try { request.execute(); } catch (BulkException ex) { final BulkResult result = ex.getBulkResult().getRawResult(); LOG.error("Bulk API action failed. Error response was: {}", result.getErrorMessage()); // Log any unsuccessful completions as errors if (LOG.isDebugEnabled() && result != null) { for (BulkResultItem item : result.getItems()) { if (item.status >= 200 && item.status < 300) { logEsDebug(item.operation, item.index, item.type, "none", item.status, item.error); } else { logEsError(item.operation, item.index, item.type, "none", item.status, item.error); } } } } catch (IOException ex) { LOG.error("Bulk API action failed. An exception occurred: {}", ex.getMessage(), ex); } }