@Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { stopProfiler(request); List<DocId> successDocIds = new ArrayList<>(); for (BulkItemResponse item : response.getItems()) { if (item.isFailed()) { LOGGER.error("index [{}], type [{}], id [{}], message [{}]", item.getIndex(), item.getType(), item.getId(), item.getFailureMessage()); } else { result.incrementSuccess(); successDocIds.add(new DocId(item.getIndex(), item.getType(), item.getId())); } } indexingListener.onSuccess(successDocIds); }
response.getType(), response.getIndex(), response.getFailureMessage() );
session.getProvenanceReporter().send(flowFile, context.getProperty(HOSTS).evaluateAttributeExpressions().getValue() + "/" + responses[i].getIndex()); session.transfer(flowFile, REL_SUCCESS);
public String buildFailureMessage() { StringBuilder sb = new StringBuilder(); sb.append("failure in bulk execution:"); for (int i = 0; i < responses.length; i++) { BulkItemResponse response = responses[i]; if (response.isFailed()) { sb.append("\n[").append(i) .append("]: index [").append(response.getIndex()).append("], type [") .append(response.getType()).append("], id [").append(response.getId()) .append("], message [").append(response.getFailureMessage()).append("]"); } } return sb.toString(); }
@Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { logger.trace("Executed bulk request with [{}] requests", request.numberOfActions()); if (response.hasFailures()) { final int[] failures = {0}; response.iterator().forEachRemaining(bir -> { if (bir.isFailed()) { failures[0]++; logger.debug("Error caught for [{}]/[{}]/[{}]: {}", bir.getIndex(), bir.getType(), bir.getId(), bir.getFailureMessage()); } }); logger.warn("Got [{}] failures of [{}] requests", failures[0], request.numberOfActions()); } }
@Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { logger.trace("Executed bulk request with [{}] requests", request.numberOfActions()); if (response.hasFailures()) { final int[] failures = {0}; response.iterator().forEachRemaining(bir -> { if (bir.isFailed()) { failures[0]++; logger.debug("Error caught for [{}]/[{}]/[{}]: {}", bir.getIndex(), bir.getType(), bir.getId(), bir.getFailureMessage()); } }); logger.warn("Got [{}] failures of [{}] requests", failures[0], request.numberOfActions()); } }
@Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { logger.trace("Executed bulk request with [{}] requests", request.numberOfActions()); if (response.hasFailures()) { final int[] failures = {0}; response.iterator().forEachRemaining(bir -> { if (bir.isFailed()) { failures[0]++; logger.debug("Error caught for [{}]/[{}]/[{}]: {}", bir.getIndex(), bir.getType(), bir.getId(), bir.getFailureMessage()); } }); logger.warn("Got [{}] failures of [{}] requests", failures[0], request.numberOfActions()); } }
public String buildFailureMessage() { StringBuilder sb = new StringBuilder(); sb.append("failure in bulk execution:"); for (int i = 0; i < responses.length; i++) { BulkItemResponse response = responses[i]; if (response.isFailed()) { sb.append("\n[").append(i) .append("]: index [").append(response.getIndex()).append("], type [").append(response.getType()).append("], id [").append(response.getId()) .append("], message [").append(response.getFailureMessage()).append("]"); } } return sb.toString(); }
public String buildFailureMessage() { StringBuilder sb = new StringBuilder(); sb.append("failure in bulk execution:"); for (int i = 0; i < responses.length; i++) { BulkItemResponse response = responses[i]; if (response.isFailed()) { sb.append("\n[").append(i) .append("]: index [").append(response.getIndex()).append("], type [").append(response.getType()).append("], id [").append(response.getId()) .append("], message [").append(response.getFailureMessage()).append("]"); } } return sb.toString(); }
public String buildFailureMessage() { StringBuilder sb = new StringBuilder(); sb.append("failure in bulk execution:"); for (int i = 0; i < responses.length; i++) { BulkItemResponse response = responses[i]; if (response.isFailed()) { sb.append("\n[").append(i) .append("]: index [").append(response.getIndex()).append("], type [").append(response.getType()).append("], id [").append(response.getId()) .append("], message [").append(response.getFailureMessage()).append("]"); } } return sb.toString(); }
public String buildFailureMessage() { StringBuilder sb = new StringBuilder(); sb.append("failure in bulk execution:"); for (int i = 0; i < responses.length; i++) { BulkItemResponse response = responses[i]; if (response.isFailed()) { sb.append("\n[").append(i) .append("]: index [").append(response.getIndex()).append("], type [").append(response.getType()).append("], id [").append(response.getId()) .append("], message [").append(response.getFailureMessage()).append("]"); } } return sb.toString(); }
private void updateTotals(final BulkResponse bulkItemResponses, final Long sent, final Long sizeInBytes) { long failed = 0; long passed = 0; long millis = bulkItemResponses.getTookInMillis(); // keep track of the number of totalFailed and items that we have totalOk. for (BulkItemResponse resp : bulkItemResponses.getItems()) { if (resp == null || resp.isFailed()) { failed++; LOGGER.debug("{} ({},{},{}) failed: {}", resp.getOpType(), resp.getIndex(), resp.getType(), resp.getId(), resp.getFailureMessage()); } else { passed++; } } if (failed > 0) { LOGGER.warn("Bulk Uploading had {} failures of {}", failed, sent); } this.totalOk.addAndGet(passed); this.totalFailed.addAndGet(failed); this.totalSeconds.addAndGet(millis / 1000); this.totalSizeInBytes.addAndGet(sizeInBytes); if (sent != (passed + failed)) { LOGGER.error("Count MisMatch: Sent[{}] Passed[{}] Failed[{}]", sent, passed, failed); } LOGGER.debug("Batch[{}mb {} items with {} failures in {}ms] - Total[{}mb {} items with {} failures in {}seconds] {} outstanding]", MEGABYTE_FORMAT.format(sizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(passed), NUMBER_FORMAT.format(failed), NUMBER_FORMAT.format(millis), MEGABYTE_FORMAT.format((double) totalSizeInBytes.get() / (double) (1024 * 1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding())); }
protected void executeBulkRequest(BulkRequestBuilder bulkRequest) { if (bulkRequest.numberOfActions() == 0) return; BulkResponse bulkResponse = bulkRequest.execute().actionGet(); if (!bulkResponse.hasFailures()) return; for (BulkItemResponse response : bulkResponse) { if (!response.isFailed()) continue; LOG.warning(String.format("Unable to save Entity %s in %s/%s, cause: %s", response.getId(), response.getIndex(), response.getType(), response.getFailureMessage())); } }
@Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { logger.trace("Executed bulk request with [{}] requests", request.numberOfActions()); if (response.hasFailures()) { final int[] failures = {0}; response.iterator().forEachRemaining(bir -> { if (bir.isFailed()) { failures[0]++; logger.debug("Error caught for [{}]/[{}]/[{}]: {}", bir.getIndex(), bir.getType(), bir.getId(), bir.getFailureMessage()); } }); logger.warn("Got [{}] failures of [{}] requests", failures[0], request.numberOfActions()); } }
@Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { stopProfiler(request); List<DocId> successDocIds = new ArrayList<>(); for (BulkItemResponse item : response.getItems()) { if (item.isFailed()) { LOGGER.error("index [{}], type [{}], id [{}], message [{}]", item.getIndex(), item.getType(), item.getId(), item.getFailureMessage()); } else { result.incrementSuccess(); successDocIds.add(new DocId(item.getIndex(), item.getType(), item.getId())); } } indexingListener.onSuccess(successDocIds); }
void onBulkResponse(String scrollId, BulkResponse bulkResponse) { try { for (BulkItemResponse item : bulkResponse.getItems()) { IndexDeleteByQueryResponse indexCounter = results.get(item.getIndex()); if (indexCounter == null) { indexCounter = new IndexDeleteByQueryResponse(item.getIndex()); } indexCounter.incrementFound(); if (item.isFailed()) { indexCounter.incrementFailed(); } else { DeleteResponse delete = item.getResponse(); if (delete.isFound()) { indexCounter.incrementDeleted(); } else { indexCounter.incrementMissing(); } } results.put(item.getIndex(), indexCounter); } logger.trace("scrolling next batch of document(s) with scroll id [{}]", scrollId); executeScroll(scrollId); } catch (Throwable t) { logger.error("unable to process bulk response", t); finishHim(scrollId, false, t); } }
/** * This method processes failures by iterating through each bulk response item * @param response, a BulkResponse **/ private ArrayList<String> processBulkResponseFailure(BulkResponse response) { ArrayList<String> urisWithESErrors = new ArrayList<String>(); logger.warn("There were failures when executing bulk : " + response.buildFailureMessage()); for(BulkItemResponse item: response.getItems()) { if (item.isFailed()) { if(logger.isDebugEnabled()){ logger.info("Error {} occurred on index {}, type {}, id {} for {} operation " , item.getFailureMessage(), item.getIndex(), item.getType(), item.getId() , item.getOpType()); } urisWithESErrors.add(String.format("%s %s", item.getId(), item.getFailureMessage())); } } return urisWithESErrors; }
builder.startObject(); builder.startObject(itemResponse.getOpType()); builder.field(Fields._INDEX, itemResponse.getIndex()); builder.field(Fields._TYPE, itemResponse.getType()); builder.field(Fields._ID, itemResponse.getId());
for (BulkItemResponse bulkItemResponse : itemResponses) { String metricId = null; String indexName = bulkItemResponse.getIndex(); String typeName = bulkItemResponse.getType(); if (bulkItemResponse.isFailed()) {
for (BulkItemResponse bulkItemResponse : itemResponses) { String metricId = null; String indexName = bulkItemResponse.getIndex(); String typeName = bulkItemResponse.getType(); if (bulkItemResponse.isFailed()) {