List<Object[]> read = myReader.read(stream).getHits(); assertEquals(7, read.size());
Scroll scroll = repository.scroll(query, body, reader); size = (size < 1 ? scroll.getTotalHits() : size); scrollId = scroll.getScrollId(); batch = scroll.getHits(); finished = scroll.isConcluded(); } catch (IOException ex) { throw new EsHadoopIllegalStateException(String.format("Cannot create scroll for query [%s/%s]", query, body), ex); scrollId = scroll.getScrollId(); batch = scroll.getHits(); finished = scroll.isConcluded(); } catch (IOException ex) { throw new EsHadoopIllegalStateException("Cannot retrieve scroll [" + scrollId + "]", ex);
return new Scroll(scrollId, results);
Scroll scroll = repository.scroll(query, body, reader); size = (size < 1 ? scroll.getTotalHits() : size); scrollId = scroll.getScrollId(); batch = scroll.getHits(); finished = scroll.isConcluded(); } catch (IOException ex) { throw new EsHadoopIllegalStateException(String.format("Cannot create scroll for query [%s/%s]", query, body), ex); scrollId = scroll.getScrollId(); batch = scroll.getHits(); finished = scroll.isConcluded(); } catch (IOException ex) { throw new EsHadoopIllegalStateException("Cannot retrieve scroll [" + scrollId + "]", ex);
Scroll scroll = repository.scroll(query, body, reader); size = (size < 1 ? scroll.getTotalHits() : size); scrollId = scroll.getScrollId(); batch = scroll.getHits(); finished = scroll.isConcluded(); } catch (IOException ex) { throw new EsHadoopIllegalStateException(String.format("Cannot create scroll for query [%s/%s]", query, body), ex); scrollId = scroll.getScrollId(); batch = scroll.getHits(); finished = scroll.isConcluded(); } catch (IOException ex) { throw new EsHadoopIllegalStateException("Cannot retrieve scroll [" + scrollId + "]", ex);
Scroll scroll = repository.scroll(query, body, reader); size = (size < 1 ? scroll.getTotalHits() : size); scrollId = scroll.getScrollId(); batch = scroll.getHits(); finished = scroll.isConcluded(); } catch (IOException ex) { throw new EsHadoopIllegalStateException(String.format("Cannot create scroll for query [%s/%s]", query, body), ex); scrollId = scroll.getScrollId(); batch = scroll.getHits(); finished = scroll.isConcluded(); } catch (IOException ex) { throw new EsHadoopIllegalStateException("Cannot retrieve scroll [" + scrollId + "]", ex);
@Override public boolean hasNext() { if (finished) return false; if (batch.isEmpty() || batchIndex >= batch.size()) { if (read >= size) { finished = true; return false; } try { Scroll scroll = repository.scroll(scrollId, reader); scrollId = scroll.getScrollId(); batch = scroll.getHits(); } catch (IOException ex) { throw new EsHadoopIllegalStateException("Cannot retrieve scroll [" + scrollId + "]", ex); } read += batch.size(); stats.docsReceived += batch.size(); if (batch.isEmpty()) { finished = true; return false; } // reset index batchIndex = 0; } return true; }
static Scroll empty(String scrollId) { return new Scroll(scrollId, 0L, true); }
static Scroll empty(String scrollId) { return new Scroll(scrollId, 0L, true); }
static Scroll empty(String scrollId) { return new Scroll(scrollId, 0L, true); }
List<Object[]> read = myReader.read(stream).getHits(); assertEquals(7, read.size());
@Test public void testScrollWithObjectFieldAndArrayIncludes() throws IOException { MappingSet mappings = getMappingSet("object-fields"); InputStream stream = getClass().getResourceAsStream(scrollData("object-fields")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_FIELD_AS_ARRAY_INCLUDE, "a.b.d:2,a.b,a.b.f"); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); ScrollReader.Scroll scroll = reader.read(stream); // First document in scroll has a single object on the nested field. assertEquals("yo", JsonUtils.query("a").get("b").get(0).get("c").apply(scroll.getHits().get(0)[1])); // Second document in scroll has a nested document field assertEquals("hello", JsonUtils.query("a").get("b").get(0).get("c").apply(scroll.getHits().get(1)[1])); // Third document in scroll has a nested document with an array field. This array field should be padded to meet // the configured array depth in the include setting above. This should be independent of the fact that the nested // object will be added to a different array further up the call stack. assertEquals("howdy", JsonUtils.query("a").get("b").get(0).get("d").get(0).get(0).apply(scroll.getHits().get(2)[1])); assertEquals("partner", JsonUtils.query("a").get("b").get(0).get("d").get(0).get(1).apply(scroll.getHits().get(2)[1])); // Fourth document has some nested arrays next to more complex datatypes assertEquals(1L, JsonUtils.query("a").get("b").get(0).get("f").get(0).apply(scroll.getHits().get(3)[1])); assertEquals(ISODateTimeFormat.dateParser().parseDateTime("2015-01-01").toDate(), JsonUtils.query("a").get("b").get(0).get("e").apply(scroll.getHits().get(3)[1])); assertEquals(3L, JsonUtils.query("a").get("b").get(1).get("f").get(0).apply(scroll.getHits().get(3)[1])); }
@Test public void testScrollWithNestedFieldAndArrayIncludes() throws IOException { MappingSet mappings = getMappingSet("nested-data"); InputStream stream = getClass().getResourceAsStream(scrollData("nested-data")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_FIELD_AS_ARRAY_INCLUDE, "a.b.d:2,a.b.f"); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); ScrollReader.Scroll scroll = reader.read(stream); // First document in scroll has a single object on the nested field. assertEquals("yo", JsonUtils.query("a").get("b").get(0).get("c").apply(scroll.getHits().get(0)[1])); // Second document in scroll has a nested document field assertEquals("hello", JsonUtils.query("a").get("b").get(0).get("c").apply(scroll.getHits().get(1)[1])); // Third document in scroll has a nested document with an array field. This array field should be padded to meet // the configured array depth in the include setting above. This should be independent of the fact that the nested // object will be added to a different array further up the call stack. assertEquals("howdy", JsonUtils.query("a").get("b").get(0).get("d").get(0).get(0).apply(scroll.getHits().get(2)[1])); assertEquals("partner", JsonUtils.query("a").get("b").get(0).get("d").get(0).get(1).apply(scroll.getHits().get(2)[1])); // Fourth document has some nested arrays next to more complex datatypes assertEquals(1L, JsonUtils.query("a").get("b").get(0).get("f").get(0).apply(scroll.getHits().get(3)[1])); assertEquals(ISODateTimeFormat.dateParser().parseDateTime("2015-01-01").toDate(), JsonUtils.query("a").get("b").get(0).get("e").apply(scroll.getHits().get(3)[1])); assertEquals(3L, JsonUtils.query("a").get("b").get(1).get("f").get(0).apply(scroll.getHits().get(3)[1])); }
@Test public void testScrollWithMultipleTypes() throws Exception { MappingSet mappings = getLegacyMappingSet("multi-type"); // Make our own scroll reader, that ignores unmapped values like the rest of the code ScrollReaderConfigBuilder scrollCfg = getScrollReaderCfg().setResolvedMapping(mappings.getResolvedView()); ScrollReader myReader = new ScrollReader(scrollCfg); InputStream stream = getClass().getResourceAsStream(scrollData("multi-type")); List<Object[]> read = myReader.read(stream).getHits(); assertEquals(3, read.size()); Object[] row1 = read.get(0); assertTrue(((Map) row1[1]).containsKey("field1")); assertEquals("value1", ((Map) row1[1]).get("field1")); assertTrue(((Map) row1[1]).containsKey("field2")); assertEquals("value2", ((Map) row1[1]).get("field2")); Object[] row2 = read.get(1); assertTrue(((Map) row2[1]).containsKey("field3")); assertEquals("value3", ((Map) row2[1]).get("field3")); Object[] row3 = read.get(2); assertTrue(((Map) row3[1]).containsKey("field4")); assertEquals("value4", ((Map) row3[1]).get("field4")); }
private RestRepository mockRepository() throws Exception { Map<String, Object> data = new HashMap<String, Object>(); data.put("field", "value"); String id = "1"; Object[] hit = new Object[]{id, data}; RestRepository mocked = Mockito.mock(RestRepository.class); ScrollReader.Scroll start = new ScrollReader.Scroll("abcd", 10, Collections.<Object[]>emptyList(), 5, 5); Mockito.doReturn(start).when(mocked).scroll(Matchers.anyString(), Matchers.any(BytesArray.class), Matchers.any(ScrollReader.class)); ScrollReader.Scroll middle = new ScrollReader.Scroll("efgh", 10, Collections.<Object[]>emptyList(), 3, 3); Mockito.doReturn(middle).when(mocked).scroll(Matchers.eq("abcd"), Matchers.any(ScrollReader.class)); ScrollReader.Scroll end = new ScrollReader.Scroll("ijkl", 10, Collections.singletonList(hit), 2, 1); Mockito.doReturn(end).when(mocked).scroll(Matchers.eq("efgh"), Matchers.any(ScrollReader.class)); ScrollReader.Scroll finalScroll = new ScrollReader.Scroll("mnop", 10, true); Mockito.doReturn(finalScroll).when(mocked).scroll(Matchers.eq("ijkl"), Matchers.any(ScrollReader.class)); RestClient mockClient = Mockito.mock(RestClient.class); Mockito.when(mockClient.deleteScroll(Matchers.eq("mnop"))).thenReturn(true); Mockito.when(mockClient.deleteScroll(Matchers.anyString())).thenReturn(false); Mockito.doReturn(mockClient).when(mocked).getRestClient(); return mocked; } }
@Test public void testScrollWithNestedArrays() throws IOException { MappingSet mappings = getMappingSet("nested-list"); InputStream stream = getClass().getResourceAsStream(scrollData("nested-list")); Settings testSettings = new TestSettings(); testSettings.setProperty(ConfigurationOptions.ES_READ_FIELD_AS_ARRAY_INCLUDE, "a:3"); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA, "" + readMetadata); testSettings.setProperty(ConfigurationOptions.ES_READ_METADATA_FIELD, "" + metadataField); testSettings.setProperty(ConfigurationOptions.ES_OUTPUT_JSON, "" + readAsJson); JdkValueReader valueReader = ObjectUtils.instantiate(JdkValueReader.class.getName(), testSettings); ScrollReader reader = new ScrollReader(ScrollReaderConfigBuilder.builder(valueReader, mappings.getResolvedView(), testSettings)); ScrollReader.Scroll scroll = reader.read(stream); // Case of already correctly nested array data assertEquals(1L, JsonUtils.query("a").get(0).get(0).get(0).apply(scroll.getHits().get(0)[1])); // Case of insufficiently nested array data assertEquals(9L, JsonUtils.query("a").get(0).get(0).get(0).apply(scroll.getHits().get(1)[1])); // Case of singleton data that is not nested in ANY array levels. assertEquals(10L, JsonUtils.query("a").get(0).get(0).get(0).apply(scroll.getHits().get(2)[1])); }