public ScrollReaderJsonTest(boolean readMetadata, String metadataField, boolean readAsJson) { this.readMetadata = readMetadata; this.readAsJson = readAsJson; this.metadataField = metadataField; this.mapper = new ObjectMapper(); reader = new ScrollReader(getScrollCfg()); }
public ScrollReaderTest(boolean readMetadata, String metadataField) { this.readMetadata = readMetadata; this.metadataField = metadataField; reader = new ScrollReader(getScrollReaderCfg()); }
@Test public void testScrollMultiValueList() throws IOException { reader = new ScrollReader(getScrollReaderCfg()); InputStream stream = getClass().getResourceAsStream(scrollData("list")); List<Object[]> read = reader.read(stream).getHits(); assertEquals(1, read.size()); Object[] objects = read.get(0); Map map = (Map) read.get(0)[1]; List links = (List) map.get("links"); assertTrue(links.contains(null)); }
@Test public void testScrollWithSource() throws IOException { reader = new ScrollReader(getScrollReaderCfg()); InputStream stream = getClass().getResourceAsStream(scrollData("source")); List<Object[]> read = reader.read(stream).getHits(); assertEquals(3, read.size()); Object[] objects = read.get(0); assertTrue(((Map) objects[1]).containsKey("source")); }
@Test public void testScrollWithNestedFields() throws IOException { InputStream stream = getClass().getResourceAsStream(mappingData("source")); MappingSet fl = parseTypelessMappings(new ObjectMapper().readValue(stream, Map.class)); ScrollReaderConfigBuilder scrollCfg = getScrollCfg().setResolvedMapping(fl.getResolvedView()); reader = new ScrollReader(scrollCfg); stream = getClass().getResourceAsStream(scrollData("source")); List<Object[]> read = reader.read(stream).getHits(); assertEquals(3, read.size()); Object[] objects = read.get(2); String doc = objects[1].toString(); Map value = mapper.readValue(doc, Map.class); assertTrue(value.containsKey("links")); // the raw json is returned which ignored mapping assertEquals("125", ((Map) value.get("links")).get("number")); }
@Test public void testScrollWithoutSource() throws IOException { reader = new ScrollReader(getScrollReaderCfg()); InputStream stream = getClass().getResourceAsStream(scrollData("empty-source")); List<Object[]> read = reader.read(stream).getHits(); assertEquals(2, read.size()); Object[] objects = read.get(0); if (readMetadata) { assertTrue(((Map) objects[1]).containsKey(metadataField)); } else { assertTrue(((Map) objects[1]).isEmpty()); } }
@Test(expected = EsHadoopParsingException.class) public void testScrollWithParsingValueException() throws IOException { InputStream stream = getClass().getResourceAsStream(mappingData("numbers-as-strings")); MappingSet fl = FieldParser.parseTypelessMappings(new ObjectMapper().readValue(stream, Map.class)); ScrollReaderConfigBuilder scrollCfg = getScrollCfg() .setResolvedMapping(fl.getResolvedView()) .setReturnRawJson(false); // parsing the doc (don't just read it as json) yields parsing exception reader = new ScrollReader(scrollCfg); stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); List<Object[]> read = reader.read(stream).getHits(); } }
@Test public void testDateMapping() throws Exception { ScrollReaderConfigBuilder scrollCfg = ScrollReaderConfigBuilder.builder(new HiveValueReader(), new TestSettings()) .setResolvedMapping(mapping("hive-date-mappingresponse.json")) .setReadMetadata(false) .setReturnRawJson(false) .setIgnoreUnmappedFields(false); ScrollReader reader = new ScrollReader(scrollCfg); InputStream stream = getClass().getResourceAsStream("hive-date-source.json"); List<Object[]> read = reader.read(stream).getHits(); assertEquals(1, read.size()); Object[] doc = read.get(0); Map map = (Map) doc[1]; assertTrue(map.containsKey(new Text("type"))); assertTrue(map.containsKey(new Text("&t"))); assertThat(map.get(new Text("&t")).toString(), containsString("2014-08-05")); }
@Test public void testScrollWithNestedFields() throws IOException { MappingSet fl = getMappingSet("source"); ScrollReaderConfigBuilder scrollReaderConfig = getScrollReaderCfg().setResolvedMapping(fl.getResolvedView()); reader = new ScrollReader(scrollReaderConfig); InputStream stream = getClass().getResourceAsStream(scrollData("source")); List<Object[]> read = reader.read(stream).getHits(); assertEquals(3, read.size()); Object[] objects = read.get(0); assertTrue(((Map) objects[1]).containsKey("source")); Map map = (Map) read.get(2)[1]; Map number = (Map) map.get("links"); Object value = number.get("number"); assertNotNull(value); assertTrue(value instanceof Short); assertEquals(Short.valueOf((short) 125), value); }
@Test public void testScrollWithMultipleTypes() throws Exception { MappingSet mappings = getLegacyMappingSet("multi-type"); // Make our own scroll reader, that ignores unmapped values like the rest of the code ScrollReaderConfigBuilder scrollCfg = getScrollReaderCfg().setResolvedMapping(mappings.getResolvedView()); ScrollReader myReader = new ScrollReader(scrollCfg); InputStream stream = getClass().getResourceAsStream(scrollData("multi-type")); List<Object[]> read = myReader.read(stream).getHits(); assertEquals(3, read.size()); Object[] row1 = read.get(0); assertTrue(((Map) row1[1]).containsKey("field1")); assertEquals("value1", ((Map) row1[1]).get("field1")); assertTrue(((Map) row1[1]).containsKey("field2")); assertEquals("value2", ((Map) row1[1]).get("field2")); Object[] row2 = read.get(1); assertTrue(((Map) row2[1]).containsKey("field3")); assertEquals("value3", ((Map) row2[1]).get("field3")); Object[] row3 = read.get(2); assertTrue(((Map) row3[1]).containsKey("field4")); assertEquals("value4", ((Map) row3[1]).get("field4")); }
@Test public void testScrollWithHandlersThatPassWithMessages() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "marco,polo,skip"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".marco" , MarcoHandler.class.getName()); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".polo" , PoloHandler.class.getName()); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".skip" , NothingToSeeHereHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); ScrollReader.Scroll scroll = reader.read(stream); assertThat(scroll.getTotalHits(), equalTo(196L)); assertThat(scroll.getHits(), is(empty())); }
@Test public void testScrollWithIgnoringHandler() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "skipskipskip"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".skipskipskip" , NothingToSeeHereHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); ScrollReader.Scroll scroll = reader.read(stream); assertThat(scroll.getTotalHits(), equalTo(196L)); assertThat(scroll.getHits(), is(empty())); }
@Test(expected = EsHadoopParsingException.class) public void testScrollWithBreakOnInvalidMapping() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); reader.read(stream); fail("Should not be able to parse string as long"); }
@Test public void testScrollWithHandlersThatCorrectsError() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "fix"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".fix" , CorrectingHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); ScrollReader.Scroll scroll = reader.read(stream); assertThat(scroll.getTotalHits(), equalTo(196L)); assertThat(scroll.getHits().size(), equalTo(1)); assertEquals(4L, JsonUtils.query("number").apply(scroll.getHits().get(0)[1])); }
@Test(expected = EsHadoopParsingException.class) public void testScrollWithThrowingAbortErrorHandler() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "throw"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".throw" , AbortingExceptionThrowingHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); reader.read(stream); fail("Should not be able to parse string as long"); }
@Test(expected = EsHadoopException.class) public void testScrollWithNeverendingHandler() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "evil"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".evil" , NeverSurrenderHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); reader.read(stream); fail("Should not be able to parse string as long"); }
@Test(expected = EsHadoopException.class) public void testScrollWithThrowingErrorHandler() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "throw"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".throw" , ExceptionThrowingHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); reader.read(stream); fail("Should not be able to parse string as long"); }
@Test public void testSimplePathReader() throws Exception { ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(new JdkValueReader(), new TestSettings()).setReadMetadata(false)); reader.read(in); }
@Test public void testSimplePathReaderJson() throws Exception { ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(new JdkValueReader(), new TestSettings()).setReadMetadata(false).setReturnRawJson(true)); reader.read(in); } }
@Test public void testScrollWithNestedArrays() throws IOException { MappingSet mappings = getMappingSet("nested-list"); InputStream stream = getClass().getResourceAsStream(scrollData("nested-list")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_FIELD_AS_ARRAY_INCLUDE, "a:3"); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); ScrollReader.Scroll scroll = reader.read(stream); // Case of already correctly nested array data assertEquals(1L, JsonUtils.query("a").get(0).get(0).get(0).apply(scroll.getHits().get(0)[1])); // Case of insufficiently nested array data assertEquals(9L, JsonUtils.query("a").get(0).get(0).get(0).apply(scroll.getHits().get(1)[1])); // Case of singleton data that is not nested in ANY array levels. assertEquals(10L, JsonUtils.query("a").get(0).get(0).get(0).apply(scroll.getHits().get(2)[1])); }