public static ScrollReaderConfigBuilder builder(ValueReader reader, Mapping resolvedMapping, Settings settings) { return builder(reader, settings).setResolvedMapping(resolvedMapping); }
@Test public void testDateMapping() throws Exception { ScrollReaderConfigBuilder scrollCfg = ScrollReaderConfigBuilder.builder(new HiveValueReader(), new TestSettings()) .setResolvedMapping(mapping("hive-date-mappingresponse.json")) .setReadMetadata(false) .setReturnRawJson(false) .setIgnoreUnmappedFields(false); ScrollReader reader = new ScrollReader(scrollCfg); InputStream stream = getClass().getResourceAsStream("hive-date-source.json"); List<Object[]> read = reader.read(stream).getHits(); assertEquals(1, read.size()); Object[] doc = read.get(0); Map map = (Map) doc[1]; assertTrue(map.containsKey(new Text("type"))); assertTrue(map.containsKey(new Text("&t"))); assertThat(map.get(new Text("&t")).toString(), containsString("2014-08-05")); }
private ScrollReaderConfigBuilder getScrollReaderCfg() { return ScrollReaderConfigBuilder.builder(new JdkValueReader(), new TestSettings()) .setReadMetadata(readMetadata) .setMetadataName(metadataField) .setReturnRawJson(readAsJson) .setIgnoreUnmappedFields(false) .setIncludeFields(Collections.<String>emptyList()) .setExcludeFields(Collections.<String>emptyList()) .setIncludeArrayFields(Collections.<String>emptyList()); }
@Test public void testScrollWithHandlersThatPassWithMessages() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "marco,polo,skip"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".marco" , MarcoHandler.class.getName()); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".polo" , PoloHandler.class.getName()); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".skip" , NothingToSeeHereHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); ScrollReader.Scroll scroll = reader.read(stream); assertThat(scroll.getTotalHits(), equalTo(196L)); assertThat(scroll.getHits(), is(empty())); }
@Test public void testScrollWithIgnoringHandler() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "skipskipskip"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".skipskipskip" , NothingToSeeHereHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); ScrollReader.Scroll scroll = reader.read(stream); assertThat(scroll.getTotalHits(), equalTo(196L)); assertThat(scroll.getHits(), is(empty())); }
@Test(expected = EsHadoopParsingException.class) public void testScrollWithBreakOnInvalidMapping() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); reader.read(stream); fail("Should not be able to parse string as long"); }
@Test(expected = EsHadoopException.class) public void testScrollWithNeverendingHandler() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "evil"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".evil" , NeverSurrenderHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); reader.read(stream); fail("Should not be able to parse string as long"); }
private ScrollReaderConfigBuilder getScrollCfg() { return ScrollReaderConfigBuilder.builder(new JdkValueReader(), new TestSettings()) .setReadMetadata(readMetadata) .setMetadataName(metadataField) .setReturnRawJson(readAsJson) .setIgnoreUnmappedFields(true); }
@Test(expected = EsHadoopParsingException.class) public void testScrollWithThrowingAbortErrorHandler() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "throw"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".throw" , AbortingExceptionThrowingHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); reader.read(stream); fail("Should not be able to parse string as long"); }
@Test(expected = EsHadoopException.class) public void testScrollWithThrowingErrorHandler() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "throw"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".throw" , ExceptionThrowingHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); reader.read(stream); fail("Should not be able to parse string as long"); }
@Test public void testScrollWithHandlersThatCorrectsError() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "fix"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".fix" , CorrectingHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); ScrollReader.Scroll scroll = reader.read(stream); assertThat(scroll.getTotalHits(), equalTo(196L)); assertThat(scroll.getHits().size(), equalTo(1)); assertEquals(4L, JsonUtils.query("number").apply(scroll.getHits().get(0)[1])); }
ScrollReaderConfigBuilder.builder(new JdkValueReader(), settings) .setReadMetadata(true) .setMetadataName("_metadata")
@Test public void testSimplePathReader() throws Exception { ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(new JdkValueReader(), new TestSettings()).setReadMetadata(false)); reader.read(in); }
@Test public void testSimplePathReaderJson() throws Exception { ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(new JdkValueReader(), new TestSettings()).setReadMetadata(false).setReturnRawJson(true)); reader.read(in); } }
ScrollReader scrollReader = new ScrollReader(ScrollReaderConfigBuilder.builder(reader, fieldMapping, settings)); if (settings.getNodesClientOnly()) { String clientNode = repository.getRestClient().getCurrentNode();
@Test public void testScrollWithNestedArrays() throws IOException { MappingSet mappings = getMappingSet("nested-list"); InputStream stream = getClass().getResourceAsStream(scrollData("nested-list")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_FIELD_AS_ARRAY_INCLUDE, "a:3"); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); ScrollReader.Scroll scroll = reader.read(stream); // Case of already correctly nested array data assertEquals(1L, JsonUtils.query("a").get(0).get(0).get(0).apply(scroll.getHits().get(0)[1])); // Case of insufficiently nested array data assertEquals(9L, JsonUtils.query("a").get(0).get(0).get(0).apply(scroll.getHits().get(1)[1])); // Case of singleton data that is not nested in ANY array levels. assertEquals(10L, JsonUtils.query("a").get(0).get(0).get(0).apply(scroll.getHits().get(2)[1])); }
@Test public void testScrollWithNestedFieldAndArrayIncludes() throws IOException { MappingSet mappings = getMappingSet("nested-data"); InputStream stream = getClass().getResourceAsStream(scrollData("nested-data")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_FIELD_AS_ARRAY_INCLUDE, "a.b.d:2,a.b.f"); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); ScrollReader.Scroll scroll = reader.read(stream); // First document in scroll has a single object on the nested field. assertEquals("yo", JsonUtils.query("a").get("b").get(0).get("c").apply(scroll.getHits().get(0)[1])); // Second document in scroll has a nested document field assertEquals("hello", JsonUtils.query("a").get("b").get(0).get("c").apply(scroll.getHits().get(1)[1])); // Third document in scroll has a nested document with an array field. This array field should be padded to meet // the configured array depth in the include setting above. This should be independent of the fact that the nested // object will be added to a different array further up the call stack. assertEquals("howdy", JsonUtils.query("a").get("b").get(0).get("d").get(0).get(0).apply(scroll.getHits().get(2)[1])); assertEquals("partner", JsonUtils.query("a").get("b").get(0).get("d").get(0).get(1).apply(scroll.getHits().get(2)[1])); // Fourth document has some nested arrays next to more complex datatypes assertEquals(1L, JsonUtils.query("a").get("b").get(0).get("f").get(0).apply(scroll.getHits().get(3)[1])); assertEquals(ISODateTimeFormat.dateParser().parseDateTime("2015-01-01").toDate(), JsonUtils.query("a").get("b").get(0).get("e").apply(scroll.getHits().get(3)[1])); assertEquals(3L, JsonUtils.query("a").get("b").get(1).get("f").get(0).apply(scroll.getHits().get(3)[1])); }
@Test public void testScrollWithObjectFieldAndArrayIncludes() throws IOException { MappingSet mappings = getMappingSet("object-fields"); InputStream stream = getClass().getResourceAsStream(scrollData("object-fields")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_FIELD_AS_ARRAY_INCLUDE, "a.b.d:2,a.b,a.b.f"); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); ScrollReader.Scroll scroll = reader.read(stream); // First document in scroll has a single object on the nested field. assertEquals("yo", JsonUtils.query("a").get("b").get(0).get("c").apply(scroll.getHits().get(0)[1])); // Second document in scroll has a nested document field assertEquals("hello", JsonUtils.query("a").get("b").get(0).get("c").apply(scroll.getHits().get(1)[1])); // Third document in scroll has a nested document with an array field. This array field should be padded to meet // the configured array depth in the include setting above. This should be independent of the fact that the nested // object will be added to a different array further up the call stack. assertEquals("howdy", JsonUtils.query("a").get("b").get(0).get("d").get(0).get(0).apply(scroll.getHits().get(2)[1])); assertEquals("partner", JsonUtils.query("a").get("b").get(0).get("d").get(0).get(1).apply(scroll.getHits().get(2)[1])); // Fourth document has some nested arrays next to more complex datatypes assertEquals(1L, JsonUtils.query("a").get("b").get(0).get("f").get(0).apply(scroll.getHits().get(3)[1])); assertEquals(ISODateTimeFormat.dateParser().parseDateTime("2015-01-01").toDate(), JsonUtils.query("a").get("b").get(0).get("e").apply(scroll.getHits().get(3)[1])); assertEquals(3L, JsonUtils.query("a").get("b").get(1).get("f").get(0).apply(scroll.getHits().get(3)[1])); }
.filters(QueryUtils.parseFilters(settings)) .fields(StringUtils.concatenate(fields, ",")); input = queryBuilder.build(client, new ScrollReader(ScrollReaderConfigBuilder.builder(new JdkValueReader(), mapping, settings)));
public static ScrollReaderConfigBuilder builder(ValueReader reader, Mapping resolvedMapping, Settings settings) { return builder(reader, settings).setResolvedMapping(resolvedMapping); }