public static <T> T instantiate(String className, Settings settings) { return instantiate(className, null, settings); }
public static <T> T instantiate(String className, ClassLoader loader, Settings settings) { T obj = instantiate(className, loader); if (obj instanceof SettingsAware) { ((SettingsAware) obj).setSettings(settings); } return obj; }
@Override protected FieldExtractor createFieldExtractor(String fieldName) { settings.setProperty(ConstantFieldExtractor.PROPERTY, fieldName); return ObjectUtils.<FieldExtractor> instantiate(settings.getMappingDefaultClassExtractor(), settings); } }
public JsonTemplatedBulk(Collection<Object> beforeObject, Collection<Object> afterObject, JsonFieldExtractors jsonExtractors, Settings settings) { super(beforeObject, afterObject, new NoOpValueWriter()); this.jsonExtractors = jsonExtractors; this.jsonWriter = ObjectUtils.instantiate(settings.getSerializerBytesConverterClassName(), settings); this.settings = settings; }
protected FieldExtractor createFieldExtractor(String fieldName) { settings.setProperty(ConstantFieldExtractor.PROPERTY, fieldName); return ObjectUtils.instantiate(settings.getMappingDefaultClassExtractor(), settings); } }
public HttpRetryHandler(Settings settings) { String retryPolicyName = settings.getBatchWriteRetryPolicy(); if (ConfigurationOptions.ES_BATCH_WRITE_RETRY_POLICY_SIMPLE.equals(retryPolicyName)) { retryPolicyName = SimpleHttpRetryPolicy.class.getName(); } else if (ConfigurationOptions.ES_BATCH_WRITE_RETRY_POLICY_NONE.equals(retryPolicyName)) { retryPolicyName = NoHttpRetryPolicy.class.getName(); } HttpRetryPolicy retryPolicy = ObjectUtils.instantiate(retryPolicyName, settings); this.retry = retryPolicy.init(); this.retryLimit = settings.getBatchWriteRetryCount(); this.retryTime = settings.getBatchWriteRetryWait(); }
private Object wrapWithFormatter(String format, final FieldExtractor createFieldExtractor) { // instantiate field extractor final IndexFormatter iformatter = ObjectUtils.instantiate(settings.getMappingIndexFormatterClassName(), settings); iformatter.configure(format); return new FieldExtractor() { @Override public Object field(Object target) { String string = createFieldExtractor.field(target).toString(); // typically a string in JSON so remove the quotes if (string.startsWith("\"")) { string = string.substring(1); } if (string.endsWith("\"")) { string = string.substring(0, string.length() - 1); } // hack: an index will always be a primitive so just call toString (instead of doing JSON parsing) // the returned value is not formatted as JSON since : 1. there's no need (it will be picked up down the chain), 2: date formatter depends on it return iformatter.format(string); } }; }
RestClient(Settings settings, NetworkClient networkClient) { this.network = networkClient; this.scrollKeepAlive = TimeValue.timeValueMillis(settings.getScrollKeepAlive()); this.indexReadMissingAsEmpty = settings.getIndexReadMissingAsEmpty(); String retryPolicyName = settings.getBatchWriteRetryPolicy(); if (ConfigurationOptions.ES_BATCH_WRITE_RETRY_POLICY_SIMPLE.equals(retryPolicyName)) { retryPolicyName = SimpleHttpRetryPolicy.class.getName(); } else if (ConfigurationOptions.ES_BATCH_WRITE_RETRY_POLICY_NONE.equals(retryPolicyName)) { retryPolicyName = NoHttpRetryPolicy.class.getName(); } this.retryPolicy = ObjectUtils.instantiate(retryPolicyName, settings); // Assume that the elasticsearch major version is the latest if the version is not already present in the settings this.internalVersion = settings.getInternalVersionOrLatest(); this.errorExtractor = new ErrorExtractor(internalVersion); }
AbstractBulkFactory(Settings settings, MetadataExtractor metaExtractor, EsMajorVersion esMajorVersion) { this.settings = settings; this.esMajorVersion = esMajorVersion; this.requestParameterNames = new RequestParameterNames(esMajorVersion); this.valueWriter = ObjectUtils.instantiate(settings.getSerializerValueWriterClassName(), settings); this.metaExtractor = metaExtractor; jsonInput = settings.getInputAsJson(); isStatic = metaExtractor == null; initExtractorsFromSettings(settings); }
@Test public void createFieldExtractor() { Settings settings = new TestSettings(); settings.setResourceWrite("test/{field}"); settings.setInternalVersion(EsMajorVersion.V_7_X); InitializationUtils.setFieldExtractorIfNotSet(settings, MapFieldExtractor.class, LOG); IndexExtractor iformat = ObjectUtils.instantiate(settings.getMappingIndexExtractorClassName(), settings); iformat.compile(new Resource(settings, false).toString()); assertThat(iformat.hasPattern(), is(true)); Map<String, String> data = new HashMap<String, String>(); data.put("field", "data"); Object field = iformat.field(data); assertThat(field.toString(), equalTo("\"_index\":\"test\",\"_type\":\"data\"")); }
@Test public void testScrollWithHandlersThatPassWithMessages() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "marco,polo,skip"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".marco" , MarcoHandler.class.getName()); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".polo" , PoloHandler.class.getName()); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".skip" , NothingToSeeHereHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); ScrollReader.Scroll scroll = reader.read(stream); assertThat(scroll.getTotalHits(), equalTo(196L)); assertThat(scroll.getHits(), is(empty())); }
@Test public void testScrollWithIgnoringHandler() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "skipskipskip"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".skipskipskip" , NothingToSeeHereHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); ScrollReader.Scroll scroll = reader.read(stream); assertThat(scroll.getTotalHits(), equalTo(196L)); assertThat(scroll.getHits(), is(empty())); }
@Test(expected = EsHadoopParsingException.class) public void testScrollWithBreakOnInvalidMapping() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); reader.read(stream); fail("Should not be able to parse string as long"); }
@Test(expected = EsHadoopException.class) public void testScrollWithThrowingErrorHandler() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "throw"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".throw" , ExceptionThrowingHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); reader.read(stream); fail("Should not be able to parse string as long"); }
@Test public void testScrollWithHandlersThatCorrectsError() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "fix"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".fix" , CorrectingHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); ScrollReader.Scroll scroll = reader.read(stream); assertThat(scroll.getTotalHits(), equalTo(196L)); assertThat(scroll.getHits().size(), equalTo(1)); assertEquals(4L, JsonUtils.query("number").apply(scroll.getHits().get(0)[1])); }
@Test(expected = EsHadoopException.class) public void testScrollWithNeverendingHandler() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "evil"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".evil" , NeverSurrenderHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); reader.read(stream); fail("Should not be able to parse string as long"); }
@Test(expected = EsHadoopParsingException.class) public void testScrollWithThrowingAbortErrorHandler() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "throw"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".throw" , AbortingExceptionThrowingHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); reader.read(stream); fail("Should not be able to parse string as long"); }
@Test(expected = EsHadoopIllegalArgumentException.class) public void createFieldExtractorNull() { Settings settings = new TestSettings(); settings.setResourceWrite("test/{field}"); settings.setInternalVersion(EsMajorVersion.V_7_X); InitializationUtils.setFieldExtractorIfNotSet(settings, MapFieldExtractor.class, LOG); IndexExtractor iformat = ObjectUtils.instantiate(settings.getMappingIndexExtractorClassName(), settings); iformat.compile(new Resource(settings, false).toString()); assertThat(iformat.hasPattern(), is(true)); Map<String, String> data = new HashMap<String, String>(); data.put("field", null); iformat.field(data); fail(); }
@Test(expected = EsHadoopIllegalArgumentException.class) public void createFieldExtractorFailure() { Settings settings = new TestSettings(); settings.setResourceWrite("test/{optional}"); settings.setInternalVersion(EsMajorVersion.V_7_X); InitializationUtils.setFieldExtractorIfNotSet(settings, MapFieldExtractor.class, LOG); IndexExtractor iformat = ObjectUtils.instantiate(settings.getMappingIndexExtractorClassName(), settings); iformat.compile(new Resource(settings, false).toString()); assertThat(iformat.hasPattern(), is(true)); Map<String, String> data = new HashMap<String, String>(); data.put("field", "data"); iformat.field(data); fail(); } }
@Test public void testScrollWithNestedArrays() throws IOException { MappingSet mappings = getMappingSet("nested-list"); InputStream stream = getClass().getResourceAsStream(scrollData("nested-list")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_FIELD_AS_ARRAY_INCLUDE, "a:3"); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); ScrollReader.Scroll scroll = reader.read(stream); // Case of already correctly nested array data assertEquals(1L, JsonUtils.query("a").get(0).get(0).get(0).apply(scroll.getHits().get(0)[1])); // Case of insufficiently nested array data assertEquals(9L, JsonUtils.query("a").get(0).get(0).get(0).apply(scroll.getHits().get(1)[1])); // Case of singleton data that is not nested in ANY array levels. assertEquals(10L, JsonUtils.query("a").get(0).get(0).get(0).apply(scroll.getHits().get(2)[1])); }