@Override public DeleteOperationResponse deleteById(String index, String type, List<String> ids) throws IOException { BulkRequest bulk = new BulkRequest(); for (int idx = 0; idx < ids.size(); idx++) { DeleteRequest request = new DeleteRequest(index, type, ids.get(idx)); bulk.add(request); } BulkResponse response = highLevelClient.bulk(bulk); DeleteOperationResponse dor = new DeleteOperationResponse(response.getTookInMillis()); return dor; }
@Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { if (response.hasFailures()) { logger.error("Bulk processor failed. {}", response.buildFailureMessage()); MongoDBRiverHelper.setRiverStatus(client, definition.getRiverName(), Status.IMPORT_FAILED); request.requests().clear(); bulkProcessor.close(); river.close(); } else { documentCount.addAndGet(response.getItems().length); logStatistics(response.getTookInMillis()); deletedDocuments.set(0); updatedDocuments.set(0); insertedDocuments.set(0); if (logger.isTraceEnabled()) { logger.trace("afterBulk - bulk [{}] success [{} items] [{} ms] total [{}]", executionId, response.getItems().length, response.getTookInMillis(), documentCount.get()); } } } };
@Override public IndexOperationResponse add(List<IndexOperationRequest> operations) throws IOException { BulkRequest bulkRequest = new BulkRequest(); for (int index = 0; index < operations.size(); index++) { IndexOperationRequest or = operations.get(index); IndexRequest indexRequest = new IndexRequest(or.getIndex(), or.getType(), or.getId()) .source(or.getFields()); bulkRequest.add(indexRequest); } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); BulkResponse response = highLevelClient.bulk(bulkRequest); IndexOperationResponse retVal = new IndexOperationResponse(response.getTookInMillis(), response.getIngestTookInMillis()); return retVal; }
@Override public void afterBulk(long executionId, BulkRequest request, BulkResponse response) { logger.info("Successful bulk {} with {} actions executed in {} ms.", executionId, request.numberOfActions(), response.getTookInMillis()); } })
@Override public void afterBulk(final long executionId, final BulkRequest request, final BulkResponse response) { final long l = outstandingBulkRequests.decrementAndGet(); final int cur = queue.addAndGet(-response.getItems().length); logger.info("Bulk actions done successfully [{}] success [{} items] [{}ms], {} outstanding bulk requests, queue size is {}", executionId, response.getItems().length, response.getTookInMillis(), l, cur); }
@Override public void onResponse(BulkResponse response) { BulkItemResponse[] items = response.getItems(); for (int i = 0; i < items.length; i++) { itemResponses.add(originalSlots[i], response.getItems()[i]); } actionListener.onResponse(new BulkResponse(itemResponses.toArray(new BulkItemResponse[itemResponses.size()]), response.getTookInMillis(), ingestTookInMillis)); }
private void updateTotals(final BulkResponse bulkItemResponses, final Long sent, final Long sizeInBytes) { long failed = 0; long passed = 0; long millis = bulkItemResponses.getTookInMillis(); // keep track of the number of totalFailed and items that we have totalOk. for (BulkItemResponse resp : bulkItemResponses.getItems()) { if (resp == null || resp.isFailed()) { failed++; LOGGER.debug("{} ({},{},{}) failed: {}", resp.getOpType(), resp.getIndex(), resp.getType(), resp.getId(), resp.getFailureMessage()); } else { passed++; } } if (failed > 0) { LOGGER.warn("Bulk Uploading had {} failures of {}", failed, sent); } this.totalOk.addAndGet(passed); this.totalFailed.addAndGet(failed); this.totalSeconds.addAndGet(millis / 1000); this.totalSizeInBytes.addAndGet(sizeInBytes); if (sent != (passed + failed)) { LOGGER.error("Count MisMatch: Sent[{}] Passed[{}] Failed[{}]", sent, passed, failed); } LOGGER.debug("Batch[{}mb {} items with {} failures in {}ms] - Total[{}mb {} items with {} failures in {}seconds] {} outstanding]", MEGABYTE_FORMAT.format(sizeInBytes / (double) (1024 * 1024)), NUMBER_FORMAT.format(passed), NUMBER_FORMAT.format(failed), NUMBER_FORMAT.format(millis), MEGABYTE_FORMAT.format((double) totalSizeInBytes.get() / (double) (1024 * 1024)), NUMBER_FORMAT.format(totalOk), NUMBER_FORMAT.format(totalFailed), NUMBER_FORMAT.format(totalSeconds), NUMBER_FORMAT.format(getTotalOutstanding())); }
@Override public DeleteOperationResponse deleteById(String index, String type, List<String> ids) throws IOException { BulkRequest bulk = new BulkRequest(); for (int idx = 0; idx < ids.size(); idx++) { DeleteRequest request = new DeleteRequest(index, type, ids.get(idx)); bulk.add(request); } BulkResponse response = highLevelClient.bulk(bulk); DeleteOperationResponse dor = new DeleteOperationResponse(response.getTookInMillis()); return dor; }
@Override public void onResponse(BulkResponse bulkResponse) { if (bulkResponse.hasFailures()) { int failedItems = 0; for (BulkItemResponse response : bulkResponse) { if (response.isFailed()) failedItems++; } if (logger.isTraceEnabled()) { logger.trace("bulk deletion failures for [{}]/[{}] items, failure message: [{}]", failedItems, bulkResponse.getItems().length, bulkResponse.buildFailureMessage()); } else { logger.error("bulk deletion failures for [{}]/[{}] items", failedItems, bulkResponse.getItems().length); } } else { logger.trace("bulk deletion took {}ms", bulkResponse.getTookInMillis()); } }
ActionListener<BulkResponse> wrapActionListenerIfNeeded(long ingestTookInMillis, ActionListener<BulkResponse> actionListener) { if (itemResponses.isEmpty()) { return ActionListener.wrap( response -> actionListener.onResponse( new BulkResponse(response.getItems(), response.getTookInMillis(), ingestTookInMillis)), actionListener::onFailure); } else { return new IngestBulkResponseListener(ingestTookInMillis, originalSlots, itemResponses, actionListener); } }
response.getTookInMillis(), response.hasFailures() ? "" : "no ")); } finally {
response.getTookInMillis(), response.hasFailures() ? "" : "no ")); } finally {
@Override public IndexOperationResponse add(List<IndexOperationRequest> operations) throws IOException { BulkRequest bulkRequest = new BulkRequest(); for (int index = 0; index < operations.size(); index++) { IndexOperationRequest or = operations.get(index); IndexRequest indexRequest = new IndexRequest(or.getIndex(), or.getType(), or.getId()) .source(or.getFields()); bulkRequest.add(indexRequest); } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); BulkResponse response = highLevelClient.bulk(bulkRequest); IndexOperationResponse retVal = new IndexOperationResponse(response.getTookInMillis(), response.getIngestTookInMillis()); return retVal; }
XContentBuilder builder = jsonBuilder(); builder.startObject(); builder.field(Fields.TOOK, response.getTookInMillis()); builder.startArray(Fields.ITEMS); for (BulkItemResponse itemResponse : response) {
@Override public void onResponse(BulkResponse bulkResponse) { if (bulkResponse.hasFailures()) { int failedItems = 0; for (BulkItemResponse response : bulkResponse) { if (response.isFailed()) failedItems++; } if (logger.isTraceEnabled()) { logger.trace("bulk deletion failures for [{}]/[{}] items, failure message: [{}]", failedItems, bulkResponse.getItems().length, bulkResponse.buildFailureMessage()); } else { logger.error("bulk deletion failures for [{}]/[{}] items", failedItems, bulkResponse.getItems().length); } } else { logger.trace("bulk deletion took " + bulkResponse.getTookInMillis() + "ms"); } }
protected void executeBulkRequest(BulkRequestBuilder bulkRequest) { BulkResponse bulkResponse = bulkRequest.execute().actionGet(); getLog().info("Bulk request of batch size: " + bulkRequest.numberOfActions() + " took " + bulkResponse.getTookInMillis() + " ms in index builder: " + getName()); for (BulkItemResponse response : bulkResponse.getItems()) { if (response.getResponse() instanceof DeleteResponse) { DeleteResponse deleteResponse = response.getResponse(); if (response.isFailed()) { getLog().error("Problem deleting doc: " + response.getId() + " in index builder: " + getName() + " error: " + response.getFailureMessage()); } else if (!deleteResponse.isFound()) { getLog().debug("ES could not find a doc with id: " + deleteResponse.getId() + " to delete in index builder: " + getName()); } else { getLog().debug("ES deleted a doc with id: " + deleteResponse.getId() + " in index builder: " + getName()); } } else if (response.getResponse() instanceof IndexResponse) { IndexResponse indexResponse = response.getResponse(); if (response.isFailed()) { getLog().error("Problem updating content for doc: " + response.getId() + " in index builder: " + getName() + " error: " + response.getFailureMessage()); } else { getLog().debug("ES indexed content for doc with id: " + indexResponse.getId() + " in index builder: " + getName()); } } } }
LOGGER.debug( "Indexing changed Entity states took {}ms", bulkResponse.getTookInMillis() );
@Override public RestResponse buildResponse(BulkResponse response, XContentBuilder builder) throws Exception { builder.startObject(); builder.field(Fields.TOOK, response.getTookInMillis()); builder.field(Fields.ERRORS, response.hasFailures()); builder.startArray(Fields.ITEMS);
try { BulkResponse bulkResponse = bulkRequest.execute().actionGet(); long bulkTime = bulkResponse.getTookInMillis(); logger.info(String.format("Insertion Elasticsearch: %dms - %d inserted - %d total", bulkTime,