@Override public IndexOperationResponse add(List<IndexOperationRequest> operations) throws IOException { BulkRequest bulkRequest = new BulkRequest(); for (int index = 0; index < operations.size(); index++) { IndexOperationRequest or = operations.get(index); IndexRequest indexRequest = new IndexRequest(or.getIndex(), or.getType(), or.getId()) .source(or.getFields()); bulkRequest.add(indexRequest); } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); BulkResponse response = highLevelClient.bulk(bulkRequest); IndexOperationResponse retVal = new IndexOperationResponse(response.getTookInMillis(), response.getIngestTookInMillis()); return retVal; }
BulkRequest getBulkRequest() { if (itemResponses.isEmpty()) { return bulkRequest; } else { BulkRequest modifiedBulkRequest = new BulkRequest(); modifiedBulkRequest.setRefreshPolicy(bulkRequest.getRefreshPolicy()); modifiedBulkRequest.waitForActiveShards(bulkRequest.waitForActiveShards()); modifiedBulkRequest.timeout(bulkRequest.timeout()); int slot = 0; List<DocWriteRequest<?>> requests = bulkRequest.requests(); originalSlots = new int[requests.size()]; // oversize, but that's ok for (int i = 0; i < requests.size(); i++) { DocWriteRequest request = requests.get(i); if (failedSlots.get(i) == false) { modifiedBulkRequest.add(request); originalSlots[slot++] = i; } } return modifiedBulkRequest; } }
@Override public BulkDocumentWriterResults<D> write() { BulkDocumentWriterResults<D> results = new BulkDocumentWriterResults<>(); try { // create an index request for each document BulkRequest bulkRequest = new BulkRequest(); bulkRequest.setRefreshPolicy(refreshPolicy); for(Indexable doc: documents) { DocWriteRequest request = createRequest(doc.document, doc.index); bulkRequest.add(request); } // submit the request and handle the response BulkResponse bulkResponse = client.getHighLevelClient().bulk(bulkRequest); handleBulkResponse(bulkResponse, documents, results); } catch(IOException e) { // assume all documents have failed for(Indexable indexable: documents) { D failed = indexable.document; results.addFailure(failed, e, ExceptionUtils.getRootCauseMessage(e)); } LOG.error("Failed to submit bulk request; all documents failed", e); } finally { // flush all documents no matter which ones succeeded or failed documents.clear(); } LOG.debug("Wrote document(s) to Elasticsearch; batchSize={}, success={}, failed={}", documents.size(), results.getSuccesses().size(), results.getFailures().size()); return results; }
public static BulkRequest toSingleItemBulkRequest(ReplicatedWriteRequest request) { BulkRequest bulkRequest = new BulkRequest(); bulkRequest.add(((DocWriteRequest) request)); bulkRequest.setRefreshPolicy(request.getRefreshPolicy()); bulkRequest.timeout(request.timeout()); bulkRequest.waitForActiveShards(request.waitForActiveShards()); request.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); return bulkRequest; } }
@Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { BulkRequest bulkRequest = Requests.bulkRequest(); String defaultIndex = request.param("index"); String defaultType = request.param("type"); String defaultRouting = request.param("routing"); FetchSourceContext defaultFetchSourceContext = FetchSourceContext.parseFromRestRequest(request); String fieldsParam = request.param("fields"); if (fieldsParam != null) { DEPRECATION_LOGGER.deprecated("Deprecated field [fields] used, expected [_source] instead"); } String[] defaultFields = fieldsParam != null ? Strings.commaDelimitedListToStringArray(fieldsParam) : null; String defaultPipeline = request.param("pipeline"); String waitForActiveShards = request.param("wait_for_active_shards"); if (waitForActiveShards != null) { bulkRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards)); } bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT)); bulkRequest.setRefreshPolicy(request.param("refresh")); bulkRequest.add(request.requiredContent(), defaultIndex, defaultType, defaultRouting, defaultFields, defaultFetchSourceContext, defaultPipeline, null, allowExplicitIndex, request.getXContentType()); return channel -> client.bulk(bulkRequest, new RestStatusToXContentListener<>(channel)); }
private BulkRequest createBulkRequest(List<DocWriteRequest> doc) { BulkRequest br = new BulkRequest(); br.add(doc); br.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); return br; }
private BulkRequest createBulkRequest(List<DocWriteRequest> doc) { BulkRequest br = new BulkRequest(); br.add(doc); br.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); br.waitForActiveShards(ActiveShardCount.ONE); return br; }
BulkRequest getBulkRequest() { if (itemResponses.isEmpty()) { return bulkRequest; } else { BulkRequest modifiedBulkRequest = new BulkRequest(); modifiedBulkRequest.setRefreshPolicy(bulkRequest.getRefreshPolicy()); modifiedBulkRequest.waitForActiveShards(bulkRequest.waitForActiveShards()); modifiedBulkRequest.timeout(bulkRequest.timeout()); int slot = 0; List<DocWriteRequest> requests = bulkRequest.requests(); originalSlots = new int[requests.size()]; // oversize, but that's ok for (int i = 0; i < requests.size(); i++) { DocWriteRequest request = requests.get(i); if (failedSlots.get(i) == false) { modifiedBulkRequest.add(request); originalSlots[slot++] = i; } } return modifiedBulkRequest; } }
BulkRequest getBulkRequest() { if (itemResponses.isEmpty()) { return bulkRequest; } else { BulkRequest modifiedBulkRequest = new BulkRequest(); modifiedBulkRequest.setRefreshPolicy(bulkRequest.getRefreshPolicy()); modifiedBulkRequest.waitForActiveShards(bulkRequest.waitForActiveShards()); modifiedBulkRequest.timeout(bulkRequest.timeout()); int slot = 0; List<DocWriteRequest<?>> requests = bulkRequest.requests(); originalSlots = new int[requests.size()]; // oversize, but that's ok for (int i = 0; i < requests.size(); i++) { DocWriteRequest request = requests.get(i); if (failedSlots.get(i) == false) { modifiedBulkRequest.add(request); originalSlots[slot++] = i; } } return modifiedBulkRequest; } }
.type(metadata.get("_type").asText()) .id(metadata.get("_id").asText()) .source(document, XContentType.JSON)).setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE);
@Override public IndexOperationResponse add(List<IndexOperationRequest> operations) throws IOException { BulkRequest bulkRequest = new BulkRequest(); for (int index = 0; index < operations.size(); index++) { IndexOperationRequest or = operations.get(index); IndexRequest indexRequest = new IndexRequest(or.getIndex(), or.getType(), or.getId()) .source(or.getFields()); bulkRequest.add(indexRequest); } bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); BulkResponse response = highLevelClient.bulk(bulkRequest); IndexOperationResponse retVal = new IndexOperationResponse(response.getTookInMillis(), response.getIngestTookInMillis()); return retVal; }
public static BulkRequest toSingleItemBulkRequest(ReplicatedWriteRequest request) { BulkRequest bulkRequest = new BulkRequest(); bulkRequest.add(((DocWriteRequest) request)); bulkRequest.setRefreshPolicy(request.getRefreshPolicy()); bulkRequest.timeout(request.timeout()); bulkRequest.waitForActiveShards(request.waitForActiveShards()); request.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); return bulkRequest; } }
BulkRequest getBulkRequest() { if (itemResponses.isEmpty()) { return bulkRequest; } else { BulkRequest modifiedBulkRequest = new BulkRequest(); modifiedBulkRequest.setRefreshPolicy(bulkRequest.getRefreshPolicy()); modifiedBulkRequest.waitForActiveShards(bulkRequest.waitForActiveShards()); modifiedBulkRequest.timeout(bulkRequest.timeout()); int slot = 0; List<DocWriteRequest> requests = bulkRequest.requests(); originalSlots = new int[requests.size()]; // oversize, but that's ok for (int i = 0; i < requests.size(); i++) { DocWriteRequest request = requests.get(i); if (failedSlots.get(i) == false) { modifiedBulkRequest.add(request); originalSlots[slot++] = i; } } return modifiedBulkRequest; } }
public static BulkRequest toSingleItemBulkRequest(ReplicatedWriteRequest request) { BulkRequest bulkRequest = new BulkRequest(); bulkRequest.add(((DocWriteRequest) request)); bulkRequest.setRefreshPolicy(request.getRefreshPolicy()); bulkRequest.timeout(request.timeout()); bulkRequest.waitForActiveShards(request.waitForActiveShards()); request.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); return bulkRequest; } }
public static BulkRequest toSingleItemBulkRequest(ReplicatedWriteRequest request) { BulkRequest bulkRequest = new BulkRequest(); bulkRequest.add(((DocWriteRequest) request)); bulkRequest.setRefreshPolicy(request.getRefreshPolicy()); bulkRequest.timeout(request.timeout()); bulkRequest.waitForActiveShards(request.waitForActiveShards()); request.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE); return bulkRequest; } }
@Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { BulkRequest bulkRequest = Requests.bulkRequest(); String defaultIndex = request.param("index"); String defaultType = request.param("type"); String defaultRouting = request.param("routing"); FetchSourceContext defaultFetchSourceContext = FetchSourceContext.parseFromRestRequest(request); String fieldsParam = request.param("fields"); if (fieldsParam != null) { DEPRECATION_LOGGER.deprecated("Deprecated field [fields] used, expected [_source] instead"); } String[] defaultFields = fieldsParam != null ? Strings.commaDelimitedListToStringArray(fieldsParam) : null; String defaultPipeline = request.param("pipeline"); String waitForActiveShards = request.param("wait_for_active_shards"); if (waitForActiveShards != null) { bulkRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards)); } bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT)); bulkRequest.setRefreshPolicy(request.param("refresh")); bulkRequest.add(request.requiredContent(), defaultIndex, defaultType, defaultRouting, defaultFields, defaultFetchSourceContext, defaultPipeline, null, allowExplicitIndex, request.getXContentType()); return channel -> client.bulk(bulkRequest, new RestStatusToXContentListener<>(channel)); }
@Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { BulkRequest bulkRequest = Requests.bulkRequest(); String defaultIndex = request.param("index"); String defaultType = request.param("type"); String defaultRouting = request.param("routing"); FetchSourceContext defaultFetchSourceContext = FetchSourceContext.parseFromRestRequest(request); String fieldsParam = request.param("fields"); if (fieldsParam != null) { DEPRECATION_LOGGER.deprecated("Deprecated field [fields] used, expected [_source] instead"); } String[] defaultFields = fieldsParam != null ? Strings.commaDelimitedListToStringArray(fieldsParam) : null; String defaultPipeline = request.param("pipeline"); String waitForActiveShards = request.param("wait_for_active_shards"); if (waitForActiveShards != null) { bulkRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards)); } bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT)); bulkRequest.setRefreshPolicy(request.param("refresh")); bulkRequest.add(request.requiredContent(), defaultIndex, defaultType, defaultRouting, defaultFields, defaultFetchSourceContext, defaultPipeline, null, allowExplicitIndex, request.getXContentType()); return channel -> client.bulk(bulkRequest, new RestStatusToXContentListener<>(channel)); }
@Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { BulkRequest bulkRequest = Requests.bulkRequest(); String defaultIndex = request.param("index"); String defaultType = request.param("type"); String defaultRouting = request.param("routing"); FetchSourceContext defaultFetchSourceContext = FetchSourceContext.parseFromRestRequest(request); String fieldsParam = request.param("fields"); if (fieldsParam != null) { DEPRECATION_LOGGER.deprecated("Deprecated field [fields] used, expected [_source] instead"); } String[] defaultFields = fieldsParam != null ? Strings.commaDelimitedListToStringArray(fieldsParam) : null; String defaultPipeline = request.param("pipeline"); String waitForActiveShards = request.param("wait_for_active_shards"); if (waitForActiveShards != null) { bulkRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards)); } bulkRequest.timeout(request.paramAsTime("timeout", BulkShardRequest.DEFAULT_TIMEOUT)); bulkRequest.setRefreshPolicy(request.param("refresh")); bulkRequest.add(request.requiredContent(), defaultIndex, defaultType, defaultRouting, defaultFields, defaultFetchSourceContext, defaultPipeline, null, allowExplicitIndex, request.getXContentType()); return channel -> client.bulk(bulkRequest, new RestStatusToXContentListener<>(channel)); }