private void updateTotals(final BulkResponse bulkItemResponses, final Long sent, final Long sizeInBytes) { long failed = 0; long passed = 0; long millis = bulkItemResponses.getTookInMillis(); // keep track of the number of totalFailed and items that we have totalOk. for (BulkItemResponse resp : bulkItemResponses.getItems()) { if (resp == null || resp.isFailed()) { failed++; LOGGER.debug("{} ({},{},{}) failed: {}", resp.getOpType(), resp.getIndex(), resp.getType(), resp.getId(), resp.getFailureMessage()); } else { passed++; } } if (failed > 0) { LOGGER.warn("Bulk Uploading had {} failures of {}", failed, sent); } this.totalOk.addAndGet(passed); this.totalFailed.addAndGet(failed); this.totalSeconds.addAndGet(millis / 1000); this.totalSizeInBytes.addAndGet(sizeInBytes); if (sent != (passed + failed)) { LOGGER.error("Count MisMatch: Sent[{}] Passed[{}] Failed[{}]", sent, passed, failed); } LOGGER.debug("Batch[{}mb {} items with {} failures in {}ms] - Total[{}mb {} items with {} failures in {}seconds] {} outstanding]", MEGABYTE_FORMAT.format(sizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(passed), NUMBER_FORMAT.format(failed), NUMBER_FORMAT.format(millis), MEGABYTE_FORMAT.format((double) totalSizeInBytes.get() / (double) (1024 * 1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding())); }
/** * . * @param response . * @return . */ public static DcBulkItemResponse getInstance(BulkItemResponse response) { if (response == null) { return null; } return new DcBulkItemResponseImpl(response.getItemId(), response.getOpType(), response.getResponse()); }
public EsBulkResponseSummary(List<BulkResponse> responses) { for (BulkResponse br : responses) { for (BulkItemResponse itemResponse : br.getItems()) { if (itemResponse.isFailed()) { failed.put(getItemId(itemResponse), itemResponse.getFailureMessage()); } else { succeeded.put(getItemId(itemResponse), itemResponse.getOpType()); } } } }
/** * This method processes failures by iterating through each bulk response item * @param response, a BulkResponse **/ private ArrayList<String> processBulkResponseFailure(BulkResponse response) { ArrayList<String> urisWithESErrors = new ArrayList<String>(); logger.warn("There were failures when executing bulk : " + response.buildFailureMessage()); for(BulkItemResponse item: response.getItems()) { if (item.isFailed()) { if(logger.isDebugEnabled()){ logger.info("Error {} occurred on index {}, type {}, id {} for {} operation " , item.getFailureMessage(), item.getIndex(), item.getType(), item.getId() , item.getOpType()); } urisWithESErrors.add(String.format("%s %s", item.getId(), item.getFailureMessage())); } } return urisWithESErrors; }
for (BulkItemResponse itemResponse : shard.getValue()) { builder.startObject(); builder.startObject(itemResponse.getOpType()); builder.field(Fields._INDEX, itemResponse.getIndex()); builder.field(Fields._TYPE, itemResponse.getType());
for (BulkItemResponse itemResponse : response) { builder.startObject(); builder.startObject(itemResponse.getOpType()); builder.field(Fields._INDEX, itemResponse.getIndex()); builder.field(Fields._TYPE, itemResponse.getType());
continue; switch (item.getOpType()) { case CREATE: case INDEX:
private void executeBlocking(final ActionRequest action) { try { if (action instanceof PutMappingRequest) { getDataContext().getElasticSearchClient().createMapping((PutMappingRequest) action); } else { final ActionResponse result = getDataContext().getElasticSearchClient().execute(action); if (result instanceof BulkResponse && ((BulkResponse) result).hasFailures()) { BulkItemResponse[] failedItems = ((BulkResponse) result).getItems(); for (int i = 0; i < failedItems.length; i++) { if (failedItems[i].isFailed()) { final BulkItemResponse failedItem = failedItems[i]; logger.error("Bulk failed with item no. {} of {}: id={} op={} status={} error={}", i + 1, failedItems.length, failedItem.getId(), failedItem.getOpType(), failedItem.status(), failedItem.getFailureMessage()); } } } } } catch (IOException e) { logger.warn("Could not execute command {} ", action, e); throw new MetaModelException("Could not execute " + action, e); } }
for (BulkItemResponse itemResponse : response) { builder.startObject(); builder.startObject(itemResponse.getOpType()); builder.field(Fields._INDEX, itemResponse.getIndex()); builder.field(Fields._TYPE, itemResponse.getType());