private static IndexRequest createIndexRequest(String element, ParameterTool parameterTool) { Map<String, Object> json = new HashMap<>(); json.put("data", element); return Requests.indexRequest() .index(parameterTool.getRequired("index")) .type(parameterTool.getRequired("type")) .id(element) .source(json); }
private static IndexRequest createIndexRequest(String element, ParameterTool parameterTool) { Map<String, Object> json = new HashMap<>(); json.put("data", element); return Requests.indexRequest() .index(parameterTool.getRequired("index")) .type(parameterTool.getRequired("type")) .id(element) .source(json); }
private static IndexRequest createIndexRequest(String element, ParameterTool parameterTool) { Map<String, Object> json = new HashMap<>(); json.put("data", element); return Requests.indexRequest() .index(parameterTool.getRequired("index")) .type(parameterTool.getRequired("type")) .id(element) .source(json); }
private static IndexRequest createIndexRequest(String element, ParameterTool parameterTool) { Map<String, Object> json = new HashMap<>(); json.put("data", element); return Requests.indexRequest() .index(parameterTool.getRequired("index")) .type(parameterTool.getRequired("type")) .id(element) .source(json); } }
private static IndexRequest createIndexRequest(String element, ParameterTool parameterTool) { Map<String, Object> json = new HashMap<>(); json.put("data", element); return Requests.indexRequest() .index(parameterTool.getRequired("index")) .type(parameterTool.getRequired("type")) .id(element) .source(json); }
public void addBulkRequest(String id, Map<?, ?> source, String routing, String parent) { bulkProcessor.add(indexRequest(index).type(type).id(id).source(source).routing(routing).parent(parent)); insertedDocuments.incrementAndGet(); }
public void addBulkRequest(String id, XContentBuilder source, String routing, String parent) { bulkProcessor.add(indexRequest(index).type(type).id(id).source(source).routing(routing).parent(parent)); insertedDocuments.incrementAndGet(); }
@Override public void process(String element, RuntimeContext ctx, RequestIndexer indexer) { Map<java.lang.String, Object> json = new HashMap<>(); json.put("data", element); indexer.add( Requests.indexRequest() .index("index") .type("type") .id("id") .source(json) ); } }
/** * Adds an index request operation to a bulk request, updating the last * timestamp for a given namespace (ie: host:dbName.collectionName) * * @param bulk */ void setLastTimestamp(final Timestamp<?> time, final BulkProcessor bulkProcessor) { try { if (logger.isTraceEnabled()) { logger.trace("setLastTimestamp [{}] [{}]", definition.getMongoOplogNamespace(), time); } bulkProcessor.add(indexRequest(definition.getRiverIndexName()).type(definition.getRiverName()) .id(definition.getMongoOplogNamespace()).source(source(time))); } catch (IOException e) { logger.error("error updating last timestamp for namespace {}", definition.getMongoOplogNamespace()); } }
final IndexRequest indexRequest = Requests.indexRequest(request.index()) .type(request.type()).id(request.id()).routing(routing).parent(parent) .source(updatedSourceAsMap, updateSourceContentType).version(updateVersion).versionType(request.versionType())
protected IndexRequest getRequest(OutgoingMessageEnvelope envelope) { String[] parts = envelope.getSystemStream().getStream().split("/"); if (parts.length != 2) { throw new SamzaException("Elasticsearch stream name must match pattern {index}/{type}"); } String index = parts[0]; String type = parts[1]; return Requests.indexRequest(index).type(type); }
public void run() { LOGGER.info("Starting thread with Data {}, batch size {} for {}", this.keys.size(), this.batchSize, this.indexName); BulkRequestBuilder bulk = getClient().prepareBulk(); for(String key : this.keys.keySet()){ try{ bulk.add(Requests.indexRequest(this.indexName).type(this.typeName) .id(key).source(this.keys.get(key))); } catch(Exception e){ LOGGER.error("{} run had an Exception {}", this.indexName, e); } } saveToEs(bulk); }
/** * Checkpointing a topic or a subscriber. The current timestamp is written * to the checkpoint index type. Note that bulk index requests are used by * checkpointing and flushCheckpoint() needs to be called after all is done. * * @param id topic or subscriber * @throws IOException if this method fails */ public void checkpoint(String id) throws IOException { indexBulk(Requests.indexRequest(pubSubIndexName).type(TYPE).id(id) .source(jsonBuilder().startObject().field("timestamp", System.currentTimeMillis()).endObject()), null); }
/** * Prepare the request for merging the existing document with a new one, can optionally detect a noop change. Returns a {@code Result} * containing a new {@code IndexRequest} to be executed on the primary and replicas. */ Result prepareUpdateIndexRequest(ShardId shardId, UpdateRequest request, GetResult getResult, boolean detectNoop) { final long updateVersion = calculateUpdateVersion(request, getResult); final IndexRequest currentRequest = request.doc(); final String routing = calculateRouting(getResult, currentRequest); final String parent = calculateParent(getResult, currentRequest); final Tuple<XContentType, Map<String, Object>> sourceAndContent = XContentHelper.convertToMap(getResult.internalSourceRef(), true); final XContentType updateSourceContentType = sourceAndContent.v1(); final Map<String, Object> updatedSourceAsMap = sourceAndContent.v2(); final boolean noop = !XContentHelper.update(updatedSourceAsMap, currentRequest.sourceAsMap(), detectNoop); // We can only actually turn the update into a noop if detectNoop is true to preserve backwards compatibility and to handle cases // where users repopulating multi-fields or adding synonyms, etc. if (detectNoop && noop) { UpdateResponse update = new UpdateResponse(shardId, getResult.getType(), getResult.getId(), getResult.getVersion(), DocWriteResponse.Result.NOOP); update.setGetResult(extractGetResult(request, request.index(), getResult.getSeqNo(), getResult.getPrimaryTerm(), getResult.getVersion(), updatedSourceAsMap, updateSourceContentType, getResult.internalSourceRef())); return new Result(update, DocWriteResponse.Result.NOOP, updatedSourceAsMap, updateSourceContentType); } else { final IndexRequest finalIndexRequest = Requests.indexRequest(request.index()) .type(request.type()).id(request.id()).routing(routing).parent(parent) .source(updatedSourceAsMap, updateSourceContentType).version(updateVersion).versionType(request.versionType()) .waitForActiveShards(request.waitForActiveShards()).timeout(request.timeout()) .setRefreshPolicy(request.getRefreshPolicy()); return new Result(finalIndexRequest, DocWriteResponse.Result.UPDATED, updatedSourceAsMap, updateSourceContentType); } }
protected IndexRequest createIndexRequest(final IndexableMailMessage message) throws IOException { final String id = (!StringUtils.isEmpty(message.getPopId()) ? message.getPopId() : message.getUid()) + "::" + message.getFolderUri(); //if(logger.isTraceEnabled()) { // logger.trace("Message: "+message.build()); //} final IndexRequest request = Requests.indexRequest(index).type(type).id(id).source(message.build()); return request; }
private void addProductToBulkRequest(RiverProduct riverProduct) { if (riverProduct.action == RiverProduct.Action.DELETE) { //bulk.add(deleteRequest(RIVER_INDEX).type(RIVER_TYPE).id(riverProduct.id)); logger.error("DELETING {}/{}/{}", RIVER_INDEX, RIVER_TYPE, riverProduct.id); client.prepareDelete(RIVER_INDEX, RIVER_TYPE, riverProduct.id).execute().actionGet(); deletedDocuments++; } else { logger.error("INDEXING {}/{}/{}", RIVER_INDEX, RIVER_TYPE, riverProduct.id); bulk.add(indexRequest(RIVER_INDEX).type(RIVER_TYPE).id(riverProduct.id).source(riverProduct.product)); insertedDocuments++; } }
private void storeLastUpdatedTimestamp(String exportTimestamp) { String json = "{ \"lastUpdatedTimestamp\" : \"" + exportTimestamp + "\" }"; IndexRequest updateTimestampRequest = indexRequest(riverIndexName).type(riverName.name()).id("lastUpdatedTimestamp").source(json); client.index(updateTimestampRequest).actionGet(); }
private void indexLine(BulkProcessor bulkProcessor, String index, String type, String line) { try { Document document = Document.fromDump(line); bulkProcessor.add(Requests.indexRequest(index == null ? document.getIndex() : index) .type(type == null ? document.getType() : type) .id(document.getId()).source(document.getDocument())); } catch(Exception e) { logger.error("Error while indexing document {}", line, e); } }
@Override public ActionFuture<IndexResponse> execute(final Client client) { final IndexRequest request = Requests.indexRequest(nodeTemplate.getIndexName()) .source(content).type("log"); return client.index(request); } });
private IndexRequest createIndexRequest(Message m) { IndexInfo info = indexInfo.create(m); if (info == null) { ++parsingFailedRowCount; return null; } else { return Requests.indexRequest(info.getIndex()) .type(info.getType()) .source(info.getSource()) .id(info.getId()) .opType(IndexRequest.OpType.CREATE); } }