@Override public String toString() { StringBuilder message = new StringBuilder().append("ES index request"); message.append(String.format(" for key '%s'", request.id())); message.append(String.format(" on index '%s'", request.index())); message.append(String.format(" on type '%s'", request.type())); return message.toString(); } }
private static IndexRequest createIndexRequest(String element, ParameterTool parameterTool) { Map<String, Object> json = new HashMap<>(); json.put("data", element); return Requests.indexRequest() .index(parameterTool.getRequired("index")) .type(parameterTool.getRequired("type")) .id(element) .source(json); }
private static IndexRequest createIndexRequest(String element, ParameterTool parameterTool) { Map<String, Object> json = new HashMap<>(); json.put("data", element); return Requests.indexRequest() .index(parameterTool.getRequired("index")) .type(parameterTool.getRequired("type")) .id(element) .source(json); } }
private static IndexRequest createIndexRequest(String element, ParameterTool parameterTool) { Map<String, Object> json = new HashMap<>(); json.put("data", element); return Requests.indexRequest() .index(parameterTool.getRequired("index")) .type(parameterTool.getRequired("type")) .id(element) .source(json); }
private static IndexRequest createIndexRequest(String element, ParameterTool parameterTool) { Map<String, Object> json = new HashMap<>(); json.put("data", element); return Requests.indexRequest() .index(parameterTool.getRequired("index")) .type(parameterTool.getRequired("type")) .id(element) .source(json); }
private static IndexRequest createIndexRequest(String element, ParameterTool parameterTool) { Map<String, Object> json = new HashMap<>(); json.put("data", element); return Requests.indexRequest() .index(parameterTool.getRequired("index")) .type(parameterTool.getRequired("type")) .id(element) .source(json); }
@Override public void process(String element, RuntimeContext ctx, RequestIndexer indexer) { Map<java.lang.String, Object> json = new HashMap<>(); json.put("data", element); indexer.add( Requests.indexRequest() .index("index") .type("type") .id("id") .source(json) ); } }
return false; ((IndexRequest) request).index(newIndices.length!=1?null:newIndices[0]); } else if (request instanceof Replaceable) { String[] newIndices = provider.provide(((Replaceable) request).indices(), request, true);
/** * Set the target index for the ReindexRequest */ public ReindexRequest setDestIndex(String destIndex) { if (destIndex != null) { this.getDestination().index(destIndex); } return this; }
public static Request index(IndexRequest indexRequest) { String method = Strings.hasLength(indexRequest.id()) ? HttpMethod.PUT.name() : HttpMethod.POST.name(); boolean isCreate = (indexRequest.opType() == DocWriteRequest.OpType.CREATE); String endpoint = endpoint(indexRequest.index(), indexRequest.type(), indexRequest.id(), isCreate ? "_create" : null); Request request = new Request(method, endpoint); Params parameters = new Params(request); parameters.withRouting(indexRequest.routing()); parameters.withTimeout(indexRequest.timeout()); parameters.withVersion(indexRequest.version()); parameters.withVersionType(indexRequest.versionType()); parameters.withPipeline(indexRequest.getPipeline()); parameters.withRefreshPolicy(indexRequest.getRefreshPolicy()); parameters.withWaitForActiveShards(indexRequest.waitForActiveShards()); BytesRef source = indexRequest.source().toBytesRef(); ContentType contentType = createContentType(indexRequest.getContentType()); request.setEntity(new ByteArrayEntity(source.bytes, source.offset, source.length, contentType)); return request; }
@Override public String toString() { StringBuilder b = new StringBuilder(); b.append("reindex from "); if (remoteInfo != null) { b.append('[').append(remoteInfo).append(']'); } searchToString(b); b.append(" to [").append(destination.index()).append(']'); if (destination.type() != null) { b.append('[').append(destination.type()).append(']'); } return b.toString(); }
BulkRequest internalAdd(IndexRequest request, @Nullable Object payload) { Objects.requireNonNull(request, "'request' must not be null"); applyGlobalMandatoryParameters(request); requests.add(request); addPayload(payload); // lack of source is validated in validate() method sizeInBytes += (request.source() != null ? request.source().length() : 0) + REQUEST_OVERHEAD; indices.add(request.index()); return this; }
private IndexRequest createRequest(D document, String index) { if(document.getTimestamp() == null) { throw new IllegalArgumentException("Document must contain the timestamp"); } // if updating an existing document, the doc ID should be defined. // if creating a new document, set the doc ID to null to allow Elasticsearch to generate one. String docId = document.getDocumentID().orElse(null); if(LOG.isDebugEnabled() && document.getDocumentID().isPresent()) { LOG.debug("Updating existing document with known doc ID; docID={}, guid={}, sensorType={}", docId, document.getGuid(), document.getSensorType()); } else if(LOG.isDebugEnabled()) { LOG.debug("Creating a new document, doc ID not yet known; guid={}, sensorType={}", document.getGuid(), document.getSensorType()); } return new IndexRequest() .source(document.getDocument()) .type(document.getSensorType() + "_doc") .index(index) .id(docId) .index(index) .timestamp(document.getTimestamp().toString()); }
void markCurrentItemAsFailed(Exception e) { IndexRequest indexRequest = getIndexWriteRequest(bulkRequest.requests().get(currentSlot)); // We hit a error during preprocessing a request, so we: // 1) Remember the request item slot from the bulk, so that we're done processing all requests we know what failed // 2) Add a bulk item failure for this request // 3) Continue with the next request in the bulk. failedSlots.set(currentSlot); BulkItemResponse.Failure failure = new BulkItemResponse.Failure(indexRequest.index(), indexRequest.type(), indexRequest.id(), e); itemResponses.add(new BulkItemResponse(currentSlot, indexRequest.opType(), failure)); }
String index = indexRequest.index(); String type = indexRequest.type(); String id = indexRequest.id(); indexRequest.index((String) metadataMap.get(IngestDocument.MetaData.INDEX)); indexRequest.type((String) metadataMap.get(IngestDocument.MetaData.TYPE)); indexRequest.id((String) metadataMap.get(IngestDocument.MetaData.ID));
void processBulkIndexIngestRequest(Task task, BulkRequest original, ActionListener<BulkResponse> listener) { long ingestStartTimeInNanos = System.nanoTime(); BulkRequestModifier bulkRequestModifier = new BulkRequestModifier(original); ingestService.executeBulkRequest(() -> bulkRequestModifier, (indexRequest, exception) -> { logger.debug(() -> new ParameterizedMessage("failed to execute pipeline [{}] for document [{}/{}/{}]", indexRequest.getPipeline(), indexRequest.index(), indexRequest.type(), indexRequest.id()), exception); bulkRequestModifier.markCurrentItemAsFailed(exception); }, (exception) -> { if (exception != null) { logger.error("failed to execute pipeline for a bulk request", exception); listener.onFailure(exception); } else { long ingestTookInMillis = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - ingestStartTimeInNanos); BulkRequest bulkRequest = bulkRequestModifier.getBulkRequest(); ActionListener<BulkResponse> actionListener = bulkRequestModifier.wrapActionListenerIfNeeded(ingestTookInMillis, listener); if (bulkRequest.requests().isEmpty()) { // at this stage, the transport bulk action can't deal with a bulk request with no requests, // so we stop and send an empty response back to the client. // (this will happen if pre-processing all items in the bulk failed) actionListener.onResponse(new BulkResponse(new BulkItemResponse[0], 0)); } else { doExecute(task, bulkRequest, actionListener); } } }, indexRequest -> bulkRequestModifier.markCurrentItemAsDropped()); }
void markCurrentItemAsDropped() { IndexRequest indexRequest = getIndexWriteRequest(bulkRequest.requests().get(currentSlot)); failedSlots.set(currentSlot); itemResponses.add( new BulkItemResponse(currentSlot, indexRequest.opType(), new UpdateResponse( new ShardId(indexRequest.index(), IndexMetaData.INDEX_UUID_NA_VALUE, 0), indexRequest.type(), indexRequest.id(), indexRequest.version(), DocWriteResponse.Result.NOOP ) ) ); }
/** Executes index operation on primary shard after updates mapping if dynamic mappings are found */ private static void executeIndexRequestOnPrimary(BulkPrimaryExecutionContext context, MappingUpdatePerformer mappingUpdater) throws Exception { final IndexRequest request = context.getRequestToExecute(); final IndexShard primary = context.getPrimary(); final SourceToParse sourceToParse = SourceToParse.source(request.index(), request.type(), request.id(), request.source(), request.getContentType()) .routing(request.routing()).parent(request.parent()); executeOnPrimaryWhileHandlingMappingUpdates(context, () -> primary.applyIndexOperationOnPrimary(request.version(), request.versionType(), sourceToParse, request.ifSeqNo(), request.ifPrimaryTerm(), request.getAutoGeneratedTimestamp(), request.isRetry()), e -> primary.getFailedIndexResult(e, request.version()), context::markOperationAsExecuted, mapping -> mappingUpdater.updateMappings(mapping, primary.shardId(), request.type())); }
indexRequest.index(request.index()) .type(request.type()).id(request.id()).setRefreshPolicy(request.getRefreshPolicy()).routing(request.routing()) .parent(request.parent()).timeout(request.timeout()).waitForActiveShards(request.waitForActiveShards())
if (destination.index() == null) { e = addValidationError("index must be specified", e); return e;