public static void uploadFile(Client tc, String filepath, String index, String id) throws Exception { LOGGER.info("Will update '" + id + "' with " + filepath); try (Reader reader = new FileReader(filepath)) { final String res = tc .index(new IndexRequest(index).type("sg").id(id).setRefreshPolicy(RefreshPolicy.IMMEDIATE) .source(id, readXContent(reader, XContentType.YAML))).actionGet().getId(); if (!id.equals(res)) { throw new Exception(" FAIL: Configuration for '" + id + "' failed for unknown reasons. Pls. consult logfile of elasticsearch"); } } catch (Exception e) { throw e; } }
private static IndexRequest createIndexRequest(String element, ParameterTool parameterTool) { Map<String, Object> json = new HashMap<>(); json.put("data", element); return Requests.indexRequest() .index(parameterTool.getRequired("index")) .type(parameterTool.getRequired("type")) .id(element) .source(json); }
private static IndexRequest newIndexRequest(ProjectMeasuresDoc doc) { String projectUuid = doc.getId(); return new IndexRequest(INDEX_TYPE_PROJECT_MEASURES.getIndex(), INDEX_TYPE_PROJECT_MEASURES.getType(), projectUuid) .routing(projectUuid) .parent(projectUuid) .source(doc.getFields()); }
@Override public IndexRequest createIndexRequest( String index, String docType, XContentType contentType, byte[] document) { return new IndexRequest(index, docType) .source(document, contentType); }
/** * Sets the content source to index. * <p> * <b>Note: the number of objects passed to this method as varargs must be an even * number. Also the first argument in each pair (the field name) must have a * valid String representation.</b> * </p> */ public IndexRequest source(XContentType xContentType, Object... source) { if (source.length % 2 != 0) { throw new IllegalArgumentException("The number of object passed must be even but was [" + source.length + "]"); } if (source.length == 2 && source[0] instanceof BytesReference && source[1] instanceof Boolean) { throw new IllegalArgumentException("you are using the removed method for source with bytes and unsafe flag, the unsafe flag" + " was removed, please just use source(BytesReference)"); } try { XContentBuilder builder = XContentFactory.contentBuilder(xContentType); builder.startObject(); for (int i = 0; i < source.length; i++) { builder.field(source[i++].toString(), source[i]); } builder.endObject(); return source(builder); } catch (IOException e) { throw new ElasticsearchGenerationException("Failed to generate", e); } }
if (updateRequest.doc() != null) { bulkContentType = enforceSameContentType(updateRequest.doc(), bulkContentType); if (updateRequest.upsertRequest() != null) { bulkContentType = enforceSameContentType(updateRequest.upsertRequest(), bulkContentType); DocWriteRequest.OpType opType = action.opType(); try (XContentBuilder metadata = XContentBuilder.builder(bulkContentType.xContent())) { metadata.startObject(); metadata.startObject(opType.getLowercase()); if (Strings.hasLength(action.index())) { metadata.field("_index", action.index()); if (Strings.hasLength(indexRequest.getPipeline())) { metadata.field("pipeline", indexRequest.getPipeline()); if (opType == DocWriteRequest.OpType.INDEX || opType == DocWriteRequest.OpType.CREATE) { IndexRequest indexRequest = (IndexRequest) action; BytesReference indexSource = indexRequest.source(); XContentType indexXContentType = indexRequest.getContentType();
/** * Checkpointing a topic or a subscriber. The current timestamp is written * to the checkpoint index type. Note that bulk index requests are used by * checkpointing and flushCheckpoint() needs to be called after all is done. * * @param id topic or subscriber * @throws IOException if this method fails */ public void checkpoint(String id) throws IOException { indexBulk(Requests.indexRequest(pubSubIndexName).type(TYPE).id(id) .source(jsonBuilder().startObject().field("timestamp", System.currentTimeMillis()).endObject()), null); }
/** * Prepare the request for merging the existing document with a new one, can optionally detect a noop change. Returns a {@code Result} * containing a new {@code IndexRequest} to be executed on the primary and replicas. */ Result prepareUpdateIndexRequest(ShardId shardId, UpdateRequest request, GetResult getResult, boolean detectNoop) { final long updateVersion = calculateUpdateVersion(request, getResult); final IndexRequest currentRequest = request.doc(); final String routing = calculateRouting(getResult, currentRequest); final String parent = calculateParent(getResult, currentRequest); final Tuple<XContentType, Map<String, Object>> sourceAndContent = XContentHelper.convertToMap(getResult.internalSourceRef(), true); final XContentType updateSourceContentType = sourceAndContent.v1(); final Map<String, Object> updatedSourceAsMap = sourceAndContent.v2(); final boolean noop = !XContentHelper.update(updatedSourceAsMap, currentRequest.sourceAsMap(), detectNoop); // We can only actually turn the update into a noop if detectNoop is true to preserve backwards compatibility and to handle cases // where users repopulating multi-fields or adding synonyms, etc. if (detectNoop && noop) { UpdateResponse update = new UpdateResponse(shardId, getResult.getType(), getResult.getId(), getResult.getVersion(), DocWriteResponse.Result.NOOP); update.setGetResult(extractGetResult(request, request.index(), getResult.getSeqNo(), getResult.getPrimaryTerm(), getResult.getVersion(), updatedSourceAsMap, updateSourceContentType, getResult.internalSourceRef())); return new Result(update, DocWriteResponse.Result.NOOP, updatedSourceAsMap, updateSourceContentType); } else { final IndexRequest finalIndexRequest = Requests.indexRequest(request.index()) .type(request.type()).id(request.id()).routing(routing).parent(parent) .source(updatedSourceAsMap, updateSourceContentType).version(updateVersion).versionType(request.versionType()) .waitForActiveShards(request.waitForActiveShards()).timeout(request.timeout()) .setRefreshPolicy(request.getRefreshPolicy()); return new Result(finalIndexRequest, DocWriteResponse.Result.UPDATED, updatedSourceAsMap, updateSourceContentType); } }
@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.startObject("source"); if (remoteInfo != null) { builder.field("remote", remoteInfo); builder.array("index", getSearchRequest().indices()); builder.startObject("dest"); builder.field("index", getDestination().index()); if (getDestination().type() != null) { builder.field("type", getDestination().type()); if (getDestination().routing() != null) { builder.field("routing", getDestination().routing()); builder.field("op_type", getDestination().opType().getLowercase()); if (getDestination().getPipeline() != null) { builder.field("pipeline", getDestination().getPipeline()); builder.field("version_type", VersionType.toString(getDestination().versionType())); builder.endObject();
@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); if (docAsUpsert) { builder.field("doc_as_upsert", docAsUpsert); XContentType xContentType = doc.getContentType(); try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, doc.source(), xContentType)) { builder.field("doc"); builder.copyCurrentStructure(parser); XContentType xContentType = upsertRequest.getContentType(); try (XContentParser parser = XContentHelper.createParser(NamedXContentRegistry.EMPTY, LoggingDeprecationHandler.INSTANCE, upsertRequest.source(), xContentType)) { builder.field("upsert"); builder.copyCurrentStructure(parser);
if (request.upsertRequest() == null && !request.docAsUpsert()) { throw new DocumentMissingException(shardId, request.type(), request.id()); IndexRequest indexRequest = request.docAsUpsert() ? request.doc() : request.upsertRequest(); case CREATE: indexRequest.source(upsertResult.v2()); break; case NONE: indexRequest.index(request.index()) .type(request.type()).id(request.id()).setRefreshPolicy(request.getRefreshPolicy()).routing(request.routing()) .parent(request.parent()).timeout(request.timeout()).waitForActiveShards(request.waitForActiveShards()) .create(true); if (request.versionType() != VersionType.INTERNAL) { indexRequest.version(request.version()).versionType(request.versionType());
public static Request index(IndexRequest indexRequest) { String method = Strings.hasLength(indexRequest.id()) ? HttpMethod.PUT.name() : HttpMethod.POST.name(); boolean isCreate = (indexRequest.opType() == DocWriteRequest.OpType.CREATE); String endpoint = endpoint(indexRequest.index(), indexRequest.type(), indexRequest.id(), isCreate ? "_create" : null); Request request = new Request(method, endpoint); Params parameters = new Params(request); parameters.withRouting(indexRequest.routing()); parameters.withTimeout(indexRequest.timeout()); parameters.withVersion(indexRequest.version()); parameters.withVersionType(indexRequest.versionType()); parameters.withPipeline(indexRequest.getPipeline()); parameters.withRefreshPolicy(indexRequest.getRefreshPolicy()); parameters.withWaitForActiveShards(indexRequest.waitForActiveShards()); BytesRef source = indexRequest.source().toBytesRef(); ContentType contentType = createContentType(indexRequest.getContentType()); request.setEntity(new ByteArrayEntity(source.bytes, source.offset, source.length, contentType)); return request; }
/** Executes index operation on primary shard after updates mapping if dynamic mappings are found */ private static void executeIndexRequestOnPrimary(BulkPrimaryExecutionContext context, MappingUpdatePerformer mappingUpdater) throws Exception { final IndexRequest request = context.getRequestToExecute(); final IndexShard primary = context.getPrimary(); final SourceToParse sourceToParse = SourceToParse.source(request.index(), request.type(), request.id(), request.source(), request.getContentType()) .routing(request.routing()).parent(request.parent()); executeOnPrimaryWhileHandlingMappingUpdates(context, () -> primary.applyIndexOperationOnPrimary(request.version(), request.versionType(), sourceToParse, request.ifSeqNo(), request.ifPrimaryTerm(), request.getAutoGeneratedTimestamp(), request.isRetry()), e -> primary.getFailedIndexResult(e, request.version()), context::markOperationAsExecuted, mapping -> mappingUpdater.updateMappings(mapping, primary.shardId(), request.type())); }
@Override public void addMessage(String queue, Message message) { Map<String, Object> doc = new HashMap<>(); doc.put("messageId", message.getId()); doc.put("payload", message.getPayload()); doc.put("queue", queue); doc.put("created", System.currentTimeMillis()); IndexRequest request = new IndexRequest(logIndexName, MSG_DOC_TYPE); request.source(doc); try { new RetryUtil<>().retryOnException( () -> elasticSearchClient.index(request).actionGet(), null, null, RETRY_COUNT, "Indexing document in for docType: message", "addMessage" ); } catch (Exception e) { logger.error("Failed to index message: {}", message.getId(), e); } }
private IndexRequest createRequest(D document, String index) { if(document.getTimestamp() == null) { throw new IllegalArgumentException("Document must contain the timestamp"); } // if updating an existing document, the doc ID should be defined. // if creating a new document, set the doc ID to null to allow Elasticsearch to generate one. String docId = document.getDocumentID().orElse(null); if(LOG.isDebugEnabled() && document.getDocumentID().isPresent()) { LOG.debug("Updating existing document with known doc ID; docID={}, guid={}, sensorType={}", docId, document.getGuid(), document.getSensorType()); } else if(LOG.isDebugEnabled()) { LOG.debug("Creating a new document, doc ID not yet known; guid={}, sensorType={}", document.getGuid(), document.getSensorType()); } return new IndexRequest() .source(document.getDocument()) .type(document.getSensorType() + "_doc") .index(index) .id(docId) .index(index) .timestamp(document.getTimestamp().toString()); }
private Mono<IndexResponse> doIndex(Object value, AdaptibleEntity<?> entity, @Nullable String index, @Nullable String type) { return Mono.defer(() -> { Object id = entity.getId(); IndexCoordinates indexCoordinates = operations.determineIndex(entity, index, type); IndexRequest request = id != null ? new IndexRequest(indexCoordinates.getIndexName(), indexCoordinates.getTypeName(), converter.convertId(id)) : new IndexRequest(indexCoordinates.getIndexName(), indexCoordinates.getTypeName()); try { request.source(resultMapper.getEntityMapper().mapToString(value), Requests.INDEX_CONTENT_TYPE); } catch (IOException e) { throw new RuntimeException(e); } if (entity.isVersionedEntity()) { Object version = entity.getVersion(); if (version != null) { request.version(((Number) version).longValue()); request.versionType(EXTERNAL); } } if (entity.hasParent()) { Object parentId = entity.getParentId(); if (parentId != null) { request.parent(converter.convertId(parentId)); } } request = prepareIndexRequest(value, request); return doIndex(request); }); }
@Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { IndexRequest indexRequest = new IndexRequest(request.param("index"), request.param("type"), request.param("id")); indexRequest.routing(request.param("routing")); indexRequest.parent(request.param("parent")); indexRequest.setPipeline(request.param("pipeline")); indexRequest.source(request.requiredContent(), request.getXContentType()); indexRequest.timeout(request.paramAsTime("timeout", IndexRequest.DEFAULT_TIMEOUT)); indexRequest.setRefreshPolicy(request.param("refresh")); indexRequest.version(RestActions.parseVersion(request)); indexRequest.versionType(VersionType.fromString(request.param("version_type"), indexRequest.versionType())); indexRequest.setIfSeqNo(request.paramAsLong("if_seq_no", indexRequest.ifSeqNo())); indexRequest.setIfPrimaryTerm(request.paramAsLong("if_primary_term", indexRequest.ifPrimaryTerm())); String sOpType = request.param("op_type"); String waitForActiveShards = request.param("wait_for_active_shards"); if (waitForActiveShards != null) { indexRequest.waitForActiveShards(ActiveShardCount.parseString(waitForActiveShards)); } if (sOpType != null) { indexRequest.opType(sOpType); } return channel -> client.index(indexRequest, new RestStatusToXContentListener<>(channel, r -> r.getLocation(indexRequest.routing()))); }
/** * Sets the doc source of the update request to be used when the document does not exists. */ public UpdateRequest upsert(XContentBuilder source) { safeUpsertRequest().source(source); return this; }
private IndexRequest newIndexRequestWithDocId(String id) { return new IndexRequest(INDEX, INDEX_TYPE_FAKE.getType()) .id(id) .source(ImmutableMap.of(FakeIndexDefinition.INT_FIELD, 42)); } }