public Resource getResourceRead() { if (resourceRead == null) { if (StringUtils.hasText(resourceSettings.getResourceRead())) { resourceRead = new Resource(resourceSettings, true); } } return resourceRead; }
public Resource getResourceWrite() { if (resourceWrite == null) { if (StringUtils.hasText(resourceSettings.getResourceWrite())) { resourceWrite = new Resource(resourceSettings, false); } } return resourceWrite; } }
public static void checkIndexNameForRead(Settings settings) { Resource readResource = new Resource(settings, true); if (readResource.index().contains("{") && readResource.index().contains("}")) { throw new EsHadoopIllegalArgumentException("Cannot read indices that have curly brace field extraction patterns in them: " + readResource.index()); } }
private Resource createResource(String target, Settings s) { s.setInternalVersion(testVersion); s.setProperty(ConfigurationOptions.ES_RESOURCE, target); return new Resource(s, true); }
@Before public void setUp() throws Exception { inputEntry = IOUtils.asString(getClass().getResourceAsStream("/org/elasticsearch/hadoop/rest/bulk-retry-input-template.json")); testSettings = new TestSettings(); testSettings.setResourceWrite("foo/bar"); testSettings.setInternalVersion(esMajorVersion); testSettings.setProperty(ConfigurationOptions.ES_BATCH_SIZE_ENTRIES, "10"); testSettings.setProperty(ConfigurationOptions.ES_BATCH_WRITE_RETRY_WAIT, "1ms"); resource = new Resource(testSettings, false); }
@Test public void testSimpleQuery() { cfg.setInternalVersion(EsMajorVersion.LATEST); cfg.setResourceRead("foo/bar"); Resource typed = new Resource(cfg, true); assertTrue(builder.resource(typed).toString().contains("foo/bar")); }
indexExtractor.setSettings(settings); indexExtractor.compile(new Resource(settings, false).toString());
@Test public void createFieldExtractor() { Settings settings = new TestSettings(); settings.setResourceWrite("test/{field}"); settings.setInternalVersion(EsMajorVersion.V_7_X); InitializationUtils.setFieldExtractorIfNotSet(settings, MapFieldExtractor.class, LOG); IndexExtractor iformat = ObjectUtils.instantiate(settings.getMappingIndexExtractorClassName(), settings); iformat.compile(new Resource(settings, false).toString()); assertThat(iformat.hasPattern(), is(true)); Map<String, String> data = new HashMap<String, String>(); data.put("field", "data"); Object field = iformat.field(data); assertThat(field.toString(), equalTo("\"_index\":\"test\",\"_type\":\"data\"")); }
@Test public void testSimpleQueryTypeless() { cfg.setInternalVersion(EsMajorVersion.LATEST); cfg.setResourceRead("foo"); Resource typeless = new Resource(cfg, true); assertTrue(builder.resource(typeless).toString().contains("foo")); assertTrue(builder.indices("foo").toString().contains("foo")); }
public static void checkIndexStatus(Settings settings) { if (!settings.getIndexReadAllowRedStatus()) { RestClient bootstrap = new RestClient(settings); Resource readResource = new Resource(settings, true); try { if (bootstrap.indexExists(readResource.index())) { RestClient.Health status = bootstrap.getHealth(readResource.index()); if (status == RestClient.Health.RED) { throw new EsHadoopIllegalStateException("Index specified [" + readResource.index() + "] is either red or " + "includes an index that is red, and thus all requested data cannot be safely and fully loaded. " + "Bailing out..."); } } } finally { bootstrap.close(); } } }
Resource resource = new Resource(settings, false);
@Test(expected = EsHadoopIllegalArgumentException.class) public void createFieldExtractorNull() { Settings settings = new TestSettings(); settings.setResourceWrite("test/{field}"); settings.setInternalVersion(EsMajorVersion.V_7_X); InitializationUtils.setFieldExtractorIfNotSet(settings, MapFieldExtractor.class, LOG); IndexExtractor iformat = ObjectUtils.instantiate(settings.getMappingIndexExtractorClassName(), settings); iformat.compile(new Resource(settings, false).toString()); assertThat(iformat.hasPattern(), is(true)); Map<String, String> data = new HashMap<String, String>(); data.put("field", null); iformat.field(data); fail(); }
@Test(expected = EsHadoopIllegalArgumentException.class) public void createFieldExtractorFailure() { Settings settings = new TestSettings(); settings.setResourceWrite("test/{optional}"); settings.setInternalVersion(EsMajorVersion.V_7_X); InitializationUtils.setFieldExtractorIfNotSet(settings, MapFieldExtractor.class, LOG); IndexExtractor iformat = ObjectUtils.instantiate(settings.getMappingIndexExtractorClassName(), settings); iformat.compile(new Resource(settings, false).toString()); assertThat(iformat.hasPattern(), is(true)); Map<String, String> data = new HashMap<String, String>(); data.put("field", "data"); iformat.field(data); fail(); } }
@Test public void testPostDocumentSuccess() throws Exception { String index = "index/type"; Settings settings = new TestSettings(); settings.setInternalVersion(EsMajorVersion.V_7_X); settings.setResourceWrite(index); Resource writeResource = new Resource(settings, false); BytesArray document = new BytesArray("{\"field\":\"value\"}"); SimpleRequest request = new SimpleRequest(Request.Method.POST, null, index, null, document); String response = "{\n" + " \"_index\": \"index\",\n" + " \"_type\": \"type\",\n" + " \"_id\": \"AbcDefGhiJklMnoPqrS_\",\n" + " \"_version\": 1,\n" + " \"result\": \"created\",\n" + " \"_shards\": {\n" + " \"total\": 2,\n" + " \"successful\": 1,\n" + " \"failed\": 0\n" + " },\n" + " \"_seq_no\": 0,\n" + " \"_primary_term\": 1\n" + "}"; NetworkClient mock = Mockito.mock(NetworkClient.class); Mockito.when(mock.execute(Mockito.eq(request))).thenReturn(new SimpleResponse(201, new FastByteArrayInputStream(new BytesArray(response)), "localhost:9200")); RestClient client = new RestClient(new TestSettings(), mock); String id = client.postDocument(writeResource, document); assertEquals("AbcDefGhiJklMnoPqrS_", id); }
@Test(expected = EsHadoopInvalidRequest.class) public void testPostDocumentWeirdness() throws Exception { String index = "index/type"; Settings settings = new TestSettings(); settings.setInternalVersion(EsMajorVersion.V_6_X); settings.setResourceWrite(index); Resource writeResource = new Resource(settings, false); BytesArray document = new BytesArray("{\"field\":\"value\"}"); SimpleRequest request = new SimpleRequest(Request.Method.POST, null, index, null, document); String response = "{\n" + " \"_index\": \"index\",\n" + " \"_type\": \"type\",\n" + " \"definitely_not_an_id\": \"AbcDefGhiJklMnoPqrS_\",\n" + // Make the ID go away " \"_version\": 1,\n" + " \"result\": \"created\",\n" + " \"_shards\": {\n" + " \"total\": 2,\n" + " \"successful\": 1,\n" + " \"failed\": 0\n" + " },\n" + " \"_seq_no\": 0,\n" + " \"_primary_term\": 1\n" + "}"; NetworkClient mock = Mockito.mock(NetworkClient.class); Mockito.when(mock.execute(Mockito.eq(request))).thenReturn(new SimpleResponse(201, new FastByteArrayInputStream(new BytesArray(response)), "localhost:9200")); RestClient client = new RestClient(new TestSettings(), mock); String id = client.postDocument(writeResource, document); assertEquals("AbcDefGhiJklMnoPqrS_", id); }
@Test public void testPostTypelessDocumentSuccess() throws Exception { String index = "index"; Settings settings = new TestSettings(); settings.setInternalVersion(EsMajorVersion.V_7_X); settings.setResourceWrite(index); Resource writeResource = new Resource(settings, false); BytesArray document = new BytesArray("{\"field\":\"value\"}"); SimpleRequest request = new SimpleRequest(Request.Method.POST, null, index + "/_doc", null, document); String response = "{\n" + " \"_index\": \"index\",\n" + " \"_type\": \"_doc\",\n" + " \"_id\": \"AbcDefGhiJklMnoPqrS_\",\n" + " \"_version\": 1,\n" + " \"result\": \"created\",\n" + " \"_shards\": {\n" + " \"total\": 2,\n" + " \"successful\": 1,\n" + " \"failed\": 0\n" + " },\n" + " \"_seq_no\": 0,\n" + " \"_primary_term\": 1\n" + "}"; NetworkClient mock = Mockito.mock(NetworkClient.class); Mockito.when(mock.execute(Mockito.eq(request))).thenReturn(new SimpleResponse(201, new FastByteArrayInputStream(new BytesArray(response)), "localhost:9200")); RestClient client = new RestClient(new TestSettings(), mock); String id = client.postDocument(writeResource, document); assertEquals("AbcDefGhiJklMnoPqrS_", id); }
@Test(expected = EsHadoopInvalidRequest.class) public void testPostTypelessDocumentWeirdness() throws Exception { String index = "index"; Settings settings = new TestSettings(); settings.setInternalVersion(EsMajorVersion.V_7_X); settings.setResourceWrite(index); Resource writeResource = new Resource(settings, false); BytesArray document = new BytesArray("{\"field\":\"value\"}"); SimpleRequest request = new SimpleRequest(Request.Method.POST, null, index + "/_doc", null, document); String response = "{\n" + " \"_index\": \"index\",\n" + " \"_type\": \"_doc\",\n" + " \"definitely_not_an_id\": \"AbcDefGhiJklMnoPqrS_\",\n" + // Make the ID go away " \"_version\": 1,\n" + " \"result\": \"created\",\n" + " \"_shards\": {\n" + " \"total\": 2,\n" + " \"successful\": 1,\n" + " \"failed\": 0\n" + " },\n" + " \"_seq_no\": 0,\n" + " \"_primary_term\": 1\n" + "}"; NetworkClient mock = Mockito.mock(NetworkClient.class); Mockito.when(mock.execute(Mockito.eq(request))).thenReturn(new SimpleResponse(201, new FastByteArrayInputStream(new BytesArray(response)), "localhost:9200")); RestClient client = new RestClient(new TestSettings(), mock); String id = client.postDocument(writeResource, document); assertEquals("AbcDefGhiJklMnoPqrS_", id); }
settings.setInternalVersion(EsMajorVersion.V_7_X); settings.setResourceWrite(index); Resource writeResource = new Resource(settings, false); BytesArray document = new BytesArray("{\"field\":\"value\"}"); SimpleRequest request = new SimpleRequest(Request.Method.POST, null, index + "/_doc", null, document);
settings.setInternalVersion(EsMajorVersion.V_6_X); settings.setResourceWrite(index); Resource writeResource = new Resource(settings, false); BytesArray document = new BytesArray("{\"field\":\"value\"}"); SimpleRequest request = new SimpleRequest(Request.Method.POST, null, index, null, document);
Resource read = new Resource(settings, true); SearchRequestBuilder queryBuilder = new SearchRequestBuilder(esVersion, settings.getReadMetadata() && settings.getReadMetadataVersion())