@Override public String toString() { StringBuilder message = new StringBuilder().append("ES index request"); message.append(String.format(" for key '%s'", request.id())); message.append(String.format(" on index '%s'", request.index())); message.append(String.format(" on type '%s'", request.type())); return message.toString(); } }
private static IndexRequest createIndexRequest(String element, ParameterTool parameterTool) { Map<String, Object> json = new HashMap<>(); json.put("data", element); return Requests.indexRequest() .index(parameterTool.getRequired("index")) .type(parameterTool.getRequired("type")) .id(element) .source(json); }
private static IndexRequest createIndexRequest(String element, ParameterTool parameterTool) { Map<String, Object> json = new HashMap<>(); json.put("data", element); return Requests.indexRequest() .index(parameterTool.getRequired("index")) .type(parameterTool.getRequired("type")) .id(element) .source(json); } }
private static IndexRequest createIndexRequest(String element, ParameterTool parameterTool) { Map<String, Object> json = new HashMap<>(); json.put("data", element); return Requests.indexRequest() .index(parameterTool.getRequired("index")) .type(parameterTool.getRequired("type")) .id(element) .source(json); }
private static IndexRequest createIndexRequest(String element, ParameterTool parameterTool) { Map<String, Object> json = new HashMap<>(); json.put("data", element); return Requests.indexRequest() .index(parameterTool.getRequired("index")) .type(parameterTool.getRequired("type")) .id(element) .source(json); }
private static IndexRequest createIndexRequest(String element, ParameterTool parameterTool) { Map<String, Object> json = new HashMap<>(); json.put("data", element); return Requests.indexRequest() .index(parameterTool.getRequired("index")) .type(parameterTool.getRequired("type")) .id(element) .source(json); }
public static void uploadFile(Client tc, String filepath, String index, String id) throws Exception { LOGGER.info("Will update '" + id + "' with " + filepath); try (Reader reader = new FileReader(filepath)) { final String res = tc .index(new IndexRequest(index).type("sg").id(id).setRefreshPolicy(RefreshPolicy.IMMEDIATE) .source(id, readXContent(reader, XContentType.YAML))).actionGet().getId(); if (!id.equals(res)) { throw new Exception(" FAIL: Configuration for '" + id + "' failed for unknown reasons. Pls. consult logfile of elasticsearch"); } } catch (Exception e) { throw e; } }
/** * Performs an index operation with a retry. * @param request The index request that we want to perform. * @param operationDescription The type of operation that we are performing. */ private void indexWithRetry(final IndexRequest request, final String operationDescription) { try { new RetryUtil<IndexResponse>().retryOnException(() -> { try { return elasticSearchClient.index(request); } catch (IOException e) { throw new RuntimeException(e); } }, null, null, RETRY_COUNT, operationDescription, "indexWithRetry"); } catch (Exception e) { Monitors.error(className, "index"); logger.error("Failed to index {} for request type: {}", request.id(), request.type(), e); } }
public void addBulkRequest(String id, XContentBuilder source, String routing, String parent) { bulkProcessor.add(indexRequest(index).type(type).id(id).source(source).routing(routing).parent(parent)); insertedDocuments.incrementAndGet(); }
public void addBulkRequest(String id, Map<?, ?> source, String routing, String parent) { bulkProcessor.add(indexRequest(index).type(type).id(id).source(source).routing(routing).parent(parent)); insertedDocuments.incrementAndGet(); }
@Override public void process(String element, RuntimeContext ctx, RequestIndexer indexer) { Map<java.lang.String, Object> json = new HashMap<>(); json.put("data", element); indexer.add( Requests.indexRequest() .index("index") .type("type") .id("id") .source(json) ); } }
/** * Adds an index request operation to a bulk request, updating the last * timestamp for a given namespace (ie: host:dbName.collectionName) * * @param bulk */ void setLastTimestamp(final Timestamp<?> time, final BulkProcessor bulkProcessor) { try { if (logger.isTraceEnabled()) { logger.trace("setLastTimestamp [{}] [{}]", definition.getMongoOplogNamespace(), time); } bulkProcessor.add(indexRequest(definition.getRiverIndexName()).type(definition.getRiverName()) .id(definition.getMongoOplogNamespace()).source(source(time))); } catch (IOException e) { logger.error("error updating last timestamp for namespace {}", definition.getMongoOplogNamespace()); } }
@Test public void testUpdate() throws Exception { final Settings settings = Settings.builder() .put(ConfigConstants.SEARCHGUARD_ROLES_MAPPING_RESOLUTION, "BOTH") .build(); setup(settings); final RestHelper rh = nonSslRestHelper(); try (TransportClient tc = getInternalTransportClient()) { tc.index(new IndexRequest("indexc").type("typec").id("0") .setRefreshPolicy(RefreshPolicy.IMMEDIATE) .source("{\"content\":1}", XContentType.JSON)).actionGet(); } HttpResponse res = rh.executePostRequest("indexc/typec/0/_update?pretty=true&refresh=true", "{\"doc\" : {\"content\":2}}", encodeBasicHeader("user_c", "user_c")); System.out.println(res.getBody()); Assert.assertEquals(HttpStatus.SC_OK, res.getStatusCode()); }
@Test public void testCCSIndexResolve() throws Exception { setup(); final RestHelper rh = nonSslRestHelper(); try (TransportClient tc = getInternalTransportClient()) { tc.index(new IndexRequest(".abc-6").type("logs").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source("{\"content\":1}", XContentType.JSON)).actionGet(); } HttpResponse res = rh.executeGetRequest("/*:.abc-6,.abc-6/_search", encodeBasicHeader("ccsresolv", "nagilum")); Assert.assertTrue(res.getBody(),res.getBody().contains("\"content\":1")); Assert.assertEquals(HttpStatus.SC_OK, res.getStatusCode()); }
@Test public void testMultiRoleSpan2() throws Exception { setup(Settings.EMPTY, new DynamicSgConfig().setSgConfig("sg_config_multirolespan.yml"), Settings.EMPTY); final RestHelper rh = nonSslRestHelper(); try (TransportClient tc = getInternalTransportClient()) { tc.index(new IndexRequest("mindex_1").type("logs").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source("{\"content\":1}", XContentType.JSON)).actionGet(); tc.index(new IndexRequest("mindex_2").type("logs").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source("{\"content\":2}", XContentType.JSON)).actionGet(); tc.index(new IndexRequest("mindex_3").type("logs").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source("{\"content\":2}", XContentType.JSON)).actionGet(); tc.index(new IndexRequest("mindex_4").type("logs").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source("{\"content\":2}", XContentType.JSON)).actionGet(); } HttpResponse res = rh.executeGetRequest("/mindex_1,mindex_2/_search", encodeBasicHeader("mindex12", "nagilum")); Assert.assertEquals(HttpStatus.SC_OK, res.getStatusCode()); res = rh.executeGetRequest("/mindex_1,mindex_3/_search", encodeBasicHeader("mindex12", "nagilum")); Assert.assertEquals(HttpStatus.SC_FORBIDDEN, res.getStatusCode()); res = rh.executeGetRequest("/mindex_1,mindex_4/_search", encodeBasicHeader("mindex12", "nagilum")); Assert.assertEquals(HttpStatus.SC_FORBIDDEN, res.getStatusCode()); }
@Test public void testRegexExcludes() throws Exception { setup(Settings.EMPTY, new DynamicSgConfig(), Settings.EMPTY); try (TransportClient tc = getInternalTransportClient(this.clusterInfo, Settings.EMPTY)) { tc.index(new IndexRequest("indexa").type("type01").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source("{\"indexa\":1}", XContentType.JSON)).actionGet(); tc.index(new IndexRequest("indexb").type("type01").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source("{\"indexb\":1}", XContentType.JSON)).actionGet(); tc.index(new IndexRequest("isallowed").type("type01").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source("{\"isallowed\":1}", XContentType.JSON)).actionGet(); tc.index(new IndexRequest("special").type("type01").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source("{\"special\":1}", XContentType.JSON)).actionGet(); tc.index(new IndexRequest("alsonotallowed").type("type01").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source("{\"alsonotallowed\":1}", XContentType.JSON)).actionGet(); } RestHelper rh = nonSslRestHelper(); Assert.assertEquals(HttpStatus.SC_OK, rh.executeGetRequest("index*/_search",encodeBasicHeader("rexclude", "nagilum")).getStatusCode()); Assert.assertEquals(HttpStatus.SC_OK, rh.executeGetRequest("indexa/_search",encodeBasicHeader("rexclude", "nagilum")).getStatusCode()); Assert.assertEquals(HttpStatus.SC_OK, rh.executeGetRequest("isallowed/_search",encodeBasicHeader("rexclude", "nagilum")).getStatusCode()); Assert.assertEquals(HttpStatus.SC_FORBIDDEN, rh.executeGetRequest("special/_search",encodeBasicHeader("rexclude", "nagilum")).getStatusCode()); Assert.assertEquals(HttpStatus.SC_FORBIDDEN, rh.executeGetRequest("alsonotallowed/_search",encodeBasicHeader("rexclude", "nagilum")).getStatusCode()); }
public static Request index(IndexRequest indexRequest) { String method = Strings.hasLength(indexRequest.id()) ? HttpMethod.PUT.name() : HttpMethod.POST.name(); boolean isCreate = (indexRequest.opType() == DocWriteRequest.OpType.CREATE); String endpoint = endpoint(indexRequest.index(), indexRequest.type(), indexRequest.id(), isCreate ? "_create" : null); Request request = new Request(method, endpoint); Params parameters = new Params(request); parameters.withRouting(indexRequest.routing()); parameters.withTimeout(indexRequest.timeout()); parameters.withVersion(indexRequest.version()); parameters.withVersionType(indexRequest.versionType()); parameters.withPipeline(indexRequest.getPipeline()); parameters.withRefreshPolicy(indexRequest.getRefreshPolicy()); parameters.withWaitForActiveShards(indexRequest.waitForActiveShards()); BytesRef source = indexRequest.source().toBytesRef(); ContentType contentType = createContentType(indexRequest.getContentType()); request.setEntity(new ByteArrayEntity(source.bytes, source.offset, source.length, contentType)); return request; }
@Test public void testDeleteByQueryDnfof() throws Exception { setup(Settings.EMPTY, new DynamicSgConfig().setSgConfig("sg_config_dnfof.yml"), Settings.EMPTY); try (TransportClient tc = getInternalTransportClient()) { for(int i=0; i<3; i++) { tc.index(new IndexRequest("vulcangov").type("kolinahr").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source("{\"content\":1}", XContentType.JSON)).actionGet(); } } RestHelper rh = nonSslRestHelper(); HttpResponse res; Assert.assertEquals(HttpStatus.SC_OK, (res=rh.executePostRequest("/vulcango*/_delete_by_query?refresh=true&wait_for_completion=true&pretty=true", "{\"query\" : {\"match_all\" : {}}}", encodeBasicHeader("nagilum", "nagilum"))).getStatusCode()); Assert.assertTrue(res.getBody().contains("\"deleted\" : 3")); }
@Test public void testSingle() throws Exception { setup(); try (TransportClient tc = getInternalTransportClient()) { tc.index(new IndexRequest("shakespeare").type("type").id("1").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source("{\"content\":1}", XContentType.JSON)).actionGet(); ConfigUpdateResponse cur = tc.execute(ConfigUpdateAction.INSTANCE, new ConfigUpdateRequest(new String[]{"config","roles","rolesmapping","internalusers","actiongroups"})).actionGet(); Assert.assertEquals(clusterInfo.numNodes, cur.getNodes().size()); } RestHelper rh = nonSslRestHelper(); //sg_shakespeare -> picard HttpResponse resc = rh.executeGetRequest("shakespeare/_search", encodeBasicHeader("picard", "picard")); System.out.println(resc.getBody()); Assert.assertEquals(HttpStatus.SC_OK, resc.getStatusCode()); Assert.assertTrue(resc.getBody().contains("\"content\":1")); resc = rh.executeHeadRequest("shakespeare", encodeBasicHeader("picard", "picard")); Assert.assertEquals(HttpStatus.SC_OK, resc.getStatusCode()); }
@Test public void testAliasResolution() throws Exception { final Settings settings = Settings.builder() .build(); setup(settings); final RestHelper rh = nonSslRestHelper(); try (TransportClient tc = getInternalTransportClient()) { tc.index(new IndexRequest("concreteindex-1").type("doc").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source("{\"content\":1}", XContentType.JSON)).actionGet(); tc.admin().indices().aliases(new IndicesAliasesRequest().addAliasAction(AliasActions.add().indices("concreteindex-1").alias("calias-1"))).actionGet(); tc.index(new IndexRequest(".kibana-6").type("doc").setRefreshPolicy(RefreshPolicy.IMMEDIATE).source("{\"content\":1}", XContentType.JSON)).actionGet(); tc.admin().indices().aliases(new IndicesAliasesRequest().addAliasAction(AliasActions.add().indices(".kibana-6").alias(".kibana"))).actionGet(); } Assert.assertEquals(HttpStatus.SC_OK, rh.executeGetRequest("calias-1/_search?pretty", encodeBasicHeader("aliastest", "nagilum")).getStatusCode()); Assert.assertEquals(HttpStatus.SC_OK, rh.executeGetRequest("calias-*/_search?pretty", encodeBasicHeader("aliastest", "nagilum")).getStatusCode()); Assert.assertEquals(HttpStatus.SC_OK, rh.executeGetRequest("*kibana/_search?pretty", encodeBasicHeader("aliastest", "nagilum")).getStatusCode()); Assert.assertEquals(HttpStatus.SC_OK, rh.executeGetRequest(".ki*ana/_search?pretty", encodeBasicHeader("aliastest", "nagilum")).getStatusCode()); Assert.assertEquals(HttpStatus.SC_OK, rh.executeGetRequest(".kibana/_search?pretty", encodeBasicHeader("aliastest", "nagilum")).getStatusCode()); }