Scroll scroll(String scrollId, ScrollReader reader) throws IOException { InputStream scroll = client.scroll(scrollId); try { return reader.read(scroll); } finally { if (scroll instanceof StatsAware) { stats.aggregate(((StatsAware) scroll).stats()); } } }
public Scroll read(InputStream content) throws IOException { Assert.notNull(content); //copy content BytesArray copy = IOUtils.asBytes(content); content = new FastByteArrayInputStream(copy); if (log.isTraceEnabled()) { log.trace("About to parse scroll content " + copy); } Parser parser = new JacksonJsonParser(content); try { return read(parser, copy); } finally { parser.close(); } }
Scroll scroll(String query, BytesArray body, ScrollReader reader) throws IOException { InputStream scroll = client.execute(POST, query, body).body(); try { return reader.read(scroll); } finally { if (scroll instanceof StatsAware) { stats.aggregate(((StatsAware) scroll).stats()); } } }
@Test public void testScrollWithMatchedQueries() throws IOException { InputStream stream = getClass().getResourceAsStream(scrollData("matched-queries")); List<Object[]> read = reader.read(stream).getHits(); assertEquals(3, read.size()); Object[] objects = read.get(0); assertTrue(((Map) objects[1]).containsKey("foo")); }
@Test public void testScrollWithFields() throws IOException { InputStream stream = getClass().getResourceAsStream(scrollData("fields")); List<Object[]> read = reader.read(stream).getHits(); assertEquals(3, read.size()); Object[] objects = read.get(0); String doc = objects[1].toString(); Map value = mapper.readValue(doc, Map.class); assertTrue(value.containsKey("fields")); if (readMetadata) { assertTrue(value.containsKey(metadataField)); assertEquals("23hrGo7VRCyao8lB9Uu5Kw", ((Map) value.get(metadataField)).get("_id")); } }
@Test public void testScrollWithMatchedQueries() throws IOException { InputStream stream = getClass().getResourceAsStream(scrollData("matched-queries")); List<Object[]> read = reader.read(stream).getHits(); assertEquals(3, read.size()); Object[] objects = read.get(1); String doc = objects[1].toString(); Map value = mapper.readValue(doc, Map.class); assertTrue(value.containsKey("links")); if (readMetadata) { assertTrue(value.containsKey(metadataField)); assertEquals(Arrays.asList("myquery"), ((Map) value.get(metadataField)).get("matched_queries")); } }
@Test public void testScrollWithSource() throws IOException { InputStream stream = getClass().getResourceAsStream(scrollData("source")); List<Object[]> read = reader.read(stream).getHits(); assertEquals(3, read.size()); Object[] objects = read.get(0); String doc = objects[1].toString(); Map value = mapper.readValue(doc, Map.class); assertEquals("source", value.get("source")); if (readMetadata) { assertTrue(value.containsKey(metadataField)); assertEquals("23hrGo7VRCyao8lB9Uu5Kw", ((Map) value.get(metadataField)).get("_id")); } }
@Test public void testScrollWithFields() throws IOException { InputStream stream = getClass().getResourceAsStream(scrollData("fields")); List<Object[]> read = reader.read(stream).getHits(); assertEquals(3, read.size()); Object[] objects = read.get(0); assertTrue(((Map) objects[1]).containsKey("fields")); }
@Test public void testScrollMultiValueList() throws IOException { reader = new ScrollReader(getScrollReaderCfg()); InputStream stream = getClass().getResourceAsStream(scrollData("list")); List<Object[]> read = reader.read(stream).getHits(); assertEquals(1, read.size()); Object[] objects = read.get(0); Map map = (Map) read.get(0)[1]; List links = (List) map.get("links"); assertTrue(links.contains(null)); }
@Test public void testScrollMultiValueList() throws IOException { InputStream stream = getClass().getResourceAsStream(scrollData("list")); List<Object[]> read = reader.read(stream).getHits(); assertEquals(1, read.size()); Object[] objects = read.get(0); String doc = objects[1].toString(); Map value = mapper.readValue(doc, Map.class); assertTrue(value.containsKey("name")); List links = (List) value.get("links"); assertTrue(links.contains(null)); if (readMetadata) { assertTrue(value.containsKey(metadataField)); assertEquals("aqOqDwYnTA29J1gfy3m8_Q", ((Map) value.get(metadataField)).get("_id")); } }
@Test public void testScrollWithSource() throws IOException { reader = new ScrollReader(getScrollReaderCfg()); InputStream stream = getClass().getResourceAsStream(scrollData("source")); List<Object[]> read = reader.read(stream).getHits(); assertEquals(3, read.size()); Object[] objects = read.get(0); assertTrue(((Map) objects[1]).containsKey("source")); }
@Test public void testScrollWithoutSource() throws IOException { reader = new ScrollReader(getScrollReaderCfg()); InputStream stream = getClass().getResourceAsStream(scrollData("empty-source")); List<Object[]> read = reader.read(stream).getHits(); assertEquals(2, read.size()); Object[] objects = read.get(0); if (readMetadata) { assertTrue(((Map) objects[1]).containsKey(metadataField)); } else { assertTrue(((Map) objects[1]).isEmpty()); } }
@Test(expected = EsHadoopParsingException.class) public void testScrollWithParsingValueException() throws IOException { InputStream stream = getClass().getResourceAsStream(mappingData("numbers-as-strings")); MappingSet fl = FieldParser.parseTypelessMappings(new ObjectMapper().readValue(stream, Map.class)); ScrollReaderConfigBuilder scrollCfg = getScrollCfg() .setResolvedMapping(fl.getResolvedView()) .setReturnRawJson(false); // parsing the doc (don't just read it as json) yields parsing exception reader = new ScrollReader(scrollCfg); stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); List<Object[]> read = reader.read(stream).getHits(); } }
@Test(expected = EsHadoopParsingException.class) public void testScrollWithBreakOnInvalidMapping() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); reader.read(stream); fail("Should not be able to parse string as long"); }
@Test public void testScrollWithHandlersThatCorrectsError() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "fix"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".fix" , CorrectingHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); ScrollReader.Scroll scroll = reader.read(stream); assertThat(scroll.getTotalHits(), equalTo(196L)); assertThat(scroll.getHits().size(), equalTo(1)); assertEquals(4L, JsonUtils.query("number").apply(scroll.getHits().get(0)[1])); }
@Test(expected = EsHadoopException.class) public void testScrollWithThrowingErrorHandler() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "throw"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".throw" , ExceptionThrowingHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); reader.read(stream); fail("Should not be able to parse string as long"); }
@Test(expected = EsHadoopException.class) public void testScrollWithNeverendingHandler() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "evil"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".evil" , NeverSurrenderHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); reader.read(stream); fail("Should not be able to parse string as long"); }
@Test(expected = EsHadoopParsingException.class) public void testScrollWithThrowingAbortErrorHandler() throws IOException { MappingSet mappings = getMappingSet("numbers-as-strings"); InputStream stream = getClass().getResourceAsStream(scrollData("numbers-as-strings")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLERS , "throw"); testSettings.setProperty(DeserializationHandlerLoader.ES_READ_DATA_ERROR_HANDLER + ".throw" , AbortingExceptionThrowingHandler.class.getName()); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); reader.read(stream); fail("Should not be able to parse string as long"); }
@Test public void testSimplePathReader() throws Exception { ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(new JdkValueReader(), new TestSettings()).setReadMetadata(false)); reader.read(in); }
@Test public void testSimplePathReaderJson() throws Exception { ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(new JdkValueReader(), new TestSettings()).setReadMetadata(false).setReturnRawJson(true)); reader.read(in); } }