@Test public void test() throws Exception { RestRepository repository = mockRepository(); ScrollReader scrollReader = Mockito.mock(ScrollReader.class); String query = "/index/type/_search?scroll=10m&etc=etc"; BytesArray body = new BytesArray("{}"); long size = 100; ScrollQuery scrollQuery = new ScrollQuery(repository, query, body, size, scrollReader); Assert.assertTrue(scrollQuery.hasNext()); Assert.assertEquals("value", JsonUtils.query("field").apply(scrollQuery.next()[1])); Assert.assertFalse(scrollQuery.hasNext()); scrollQuery.close(); Stats stats = scrollQuery.stats(); Assert.assertEquals(1, stats.docsReceived); }
@Override public boolean next(K key, V value) throws IOException { if (scrollQuery == null) { if (beat != null) { beat.start(); } scrollQuery = queryBuilder.build(client, scrollReader); size = scrollQuery.getSize(); if (log.isTraceEnabled()) { log.trace(String.format("Received scroll [%s], size [%d] for query [%s]", scrollQuery, size, queryBuilder)); } } boolean hasNext = scrollQuery.hasNext(); if (!hasNext) { return false; } Object[] next = scrollQuery.next(); // NB: the left assignment is not needed since method override // the writable content however for consistency, they are below currentKey = setCurrentKey(key, next[0]); currentValue = setCurrentValue(value, next[1]); // keep on counting read++; return true; }
@Override public void sourceCleanup(FlowProcess<Properties> flowProcess, SourceCall<Object[], ScrollQuery> sourceCall) throws IOException { // in case of a source there's no local client so do all reporting here report(sourceCall.getInput().stats(), flowProcess); report(sourceCall.getInput().repository().stats(), flowProcess); sourceCall.getInput().close(); sourceCall.setContext(null); // used for consistency cleanupClient(flowProcess); }
private ScrollQuery getCurrent() { if (finished) { return null; } for (boolean hasValue = false; !hasValue; ) { if (currentReader == null) { if (definitionIterator.hasNext()) { currentReader = RestService.createReader(settings, definitionIterator.next(), log); } else { finished = true; return null; } } if (currentScroll == null) { currentScroll = currentReader.scrollQuery(); } hasValue = currentScroll.hasNext(); if (!hasValue) { currentScroll.close(); currentScroll = null; currentReader.close(); currentReader = null; } } return currentScroll; }
@Override public void close() throws IOException { try { if (log.isDebugEnabled()) { log.debug(String.format("Closing RecordReader for [%s]", esSplit)); } if (beat != null) { beat.stop(); } if (scrollQuery != null) { scrollQuery.close(); } if (client != null) { client.close(); } } finally { Stats stats = new Stats(); if (client != null) { stats.aggregate(client.stats()); client = null; } if (scrollQuery != null) { stats.aggregate(scrollQuery.stats()); scrollQuery = null; } ReportingUtils.report(progressable, stats); } }
@Override public Object[] next() { if (!hasNext()) { throw new NoSuchElementException("No more documents available"); } return batch.get(batchIndex++); }
@Override public void close() { if (!closed) { closed = true; if (scrollQuery != null) { scrollQuery.close(); } client.close(); } }
@Override public Object[] next() { ScrollQuery sq = getCurrent(); return sq.next(); }
/** * Returns a pageable (scan based) result to the given query. * * @param query scan query * @param reader scroll reader * @return a scroll query */ ScrollQuery scanLimit(String query, BytesArray body, long limit, ScrollReader reader) { return new ScrollQuery(this, query, body, limit, reader); }
private ScrollQuery getCurrent() { if (finished) { return null; } for (boolean hasValue = false; !hasValue; ) { if (currentReader == null) { if (definitionIterator.hasNext()) { currentReader = RestService.createReader(settings, definitionIterator.next(), log); } else { finished = true; return null; } } if (currentScroll == null) { currentScroll = currentReader.scrollQuery(); } hasValue = currentScroll.hasNext(); if (!hasValue) { currentScroll.close(); currentScroll = null; currentReader.close(); currentReader = null; } } return currentScroll; }
@Override public void close() throws IOException { try { if (log.isDebugEnabled()) { log.debug(String.format("Closing RecordReader for [%s]", esSplit)); } if (beat != null) { beat.stop(); } if (scrollQuery != null) { scrollQuery.close(); } if (client != null) { client.close(); } } finally { Stats stats = new Stats(); if (client != null) { stats.aggregate(client.stats()); client = null; } if (scrollQuery != null) { stats.aggregate(scrollQuery.stats()); scrollQuery = null; } ReportingUtils.report(progressable, stats); } }
@Override public boolean hasNext() { ScrollQuery sq = getCurrent(); return (sq != null ? sq.hasNext() : false); }
@Override public void close() { if (finished) { return; } ScrollQuery sq = getCurrent(); if (sq != null) { sq.close(); } if (currentReader != null) { currentReader.close(); } finished = true; }
@Override public Object[] next() { ScrollQuery sq = getCurrent(); return sq.next(); }
/** * Returns a pageable (scan based) result to the given query. * * @param query scan query * @param reader scroll reader * @return a scroll query */ ScrollQuery scanLimit(String query, BytesArray body, long limit, ScrollReader reader) { return new ScrollQuery(this, query, body, limit, reader); }
@Override public boolean next(K key, V value) throws IOException { if (scrollQuery == null) { if (beat != null) { beat.start(); } scrollQuery = queryBuilder.build(client, scrollReader); size = scrollQuery.getSize(); if (log.isTraceEnabled()) { log.trace(String.format("Received scroll [%s], size [%d] for query [%s]", scrollQuery, size, queryBuilder)); } } boolean hasNext = scrollQuery.hasNext(); if (!hasNext) { return false; } Object[] next = scrollQuery.next(); // NB: the left assignment is not needed since method override // the writable content however for consistency, they are below currentKey = setCurrentKey(key, next[0]); currentValue = setCurrentValue(value, next[1]); // keep on counting read++; return true; }
private ScrollQuery getCurrent() { if (finished) { return null; } for (boolean hasValue = false; !hasValue; ) { if (currentReader == null) { if (definitionIterator.hasNext()) { currentReader = RestService.createReader(settings, definitionIterator.next(), log); } else { finished = true; return null; } } if (currentScroll == null) { currentScroll = currentReader.scrollQuery(); } hasValue = currentScroll.hasNext(); if (!hasValue) { currentScroll.close(); currentScroll = null; currentReader.close(); currentReader = null; } } return currentScroll; }