@Override public boolean hasMore() { return firehose.hasMore(); }
@Override public boolean hasMore() { return currentFirehose.hasMore(); }
@Override public boolean hasMore() { return i < count && delegateFirehose.hasMore(); }
@Nullable @Override public InputRow nextRow() { InputRow rv = currentFirehose.nextRow(); if (!currentFirehose.hasMore()) { nextFirehose(); } return rv; }
private boolean runFirehose(Firehose firehose) { final Supplier<Committer> committerSupplier = Committers.supplierFromFirehose(firehose); while (firehose.hasMore()) { if (Thread.interrupted() || stopping) { return false; } Plumbers.addNextRow(committerSupplier, firehose, plumber, config.isReportParseExceptions(), metrics); } return true; }
@Override public boolean hasMore() { if (savedInputRow != null) { return true; } while (firehose.hasMore()) { final InputRow row = firehose.nextRow(); if (predicate.apply(row)) { savedInputRow = row; return true; } // Do not silently discard the rows if (ignored % IGNORE_THRESHOLD == 0) { log.warn("[%,d] InputRow(s) ignored as they do not satisfy the predicate", ignored); } ignored++; } return false; }
@Test public void testWithLargeCacheAndSmallFetch() throws IOException { final TestPrefetchableTextFilesFirehoseFactory factory = TestPrefetchableTextFilesFirehoseFactory.with(TEST_DIR, 2048, 1024); final List<Row> rows = new ArrayList<>(); final File firehoseTmpDir = createFirehoseTmpDir("testWithLargeCacheAndSmallFetch"); try (Firehose firehose = factory.connect(parser, firehoseTmpDir)) { while (firehose.hasMore()) { rows.add(firehose.nextRow()); } } assertResult(rows); assertNumRemainingCacheFiles(firehoseTmpDir, 2); }
@Test public void testWithCacheAndFetch() throws IOException { final TestPrefetchableTextFilesFirehoseFactory factory = TestPrefetchableTextFilesFirehoseFactory.of(TEST_DIR); final List<Row> rows = new ArrayList<>(); final File firehoseTmpDir = createFirehoseTmpDir("testWithCacheAndFetch"); try (Firehose firehose = factory.connect(parser, firehoseTmpDir)) { while (firehose.hasMore()) { rows.add(firehose.nextRow()); } } assertResult(rows); assertNumRemainingCacheFiles(firehoseTmpDir, 2); }
@Test public void testWithSmallCacheAndLargeFetch() throws IOException { final TestPrefetchableTextFilesFirehoseFactory factory = TestPrefetchableTextFilesFirehoseFactory.with(TEST_DIR, 1024, 2048); final List<Row> rows = new ArrayList<>(); final File firehoseTmpDir = createFirehoseTmpDir("testWithSmallCacheAndLargeFetch"); try (Firehose firehose = factory.connect(parser, firehoseTmpDir)) { while (firehose.hasMore()) { rows.add(firehose.nextRow()); } } assertResult(rows); assertNumRemainingCacheFiles(firehoseTmpDir, 1); }
@Test public void testRetry() throws IOException { final TestPrefetchableTextFilesFirehoseFactory factory = TestPrefetchableTextFilesFirehoseFactory.withOpenExceptions(TEST_DIR, 1); final List<Row> rows = new ArrayList<>(); final File firehoseTmpDir = createFirehoseTmpDir("testRetry"); try (Firehose firehose = factory.connect(parser, firehoseTmpDir)) { while (firehose.hasMore()) { rows.add(firehose.nextRow()); } } assertResult(rows); assertNumRemainingCacheFiles(firehoseTmpDir, 2); }
@Test public void testWithZeroFetchCapacity() throws IOException { final TestPrefetchableTextFilesFirehoseFactory factory = TestPrefetchableTextFilesFirehoseFactory.with(TEST_DIR, 2048, 0); final List<Row> rows = new ArrayList<>(); final File firehoseTmpDir = createFirehoseTmpDir("testWithZeroFetchCapacity"); try (Firehose firehose = factory.connect(parser, firehoseTmpDir)) { while (firehose.hasMore()) { rows.add(firehose.nextRow()); } } assertResult(rows); assertNumRemainingCacheFiles(firehoseTmpDir, 2); }
@Test public void testSimple() throws Exception { int count = 0; long sum = 0; try (final Firehose firehose = factory.connect(ROW_PARSER, null)) { while (firehose.hasMore()) { final InputRow row = firehose.nextRow(); count++; sum += row.getMetric(METRICS[0]).longValue(); } } Assert.assertEquals("count", expectedCount, count); Assert.assertEquals("sum", expectedSum, sum); }
@Test public void testTimeout() throws IOException { expectedException.expect(RuntimeException.class); expectedException.expectCause(CoreMatchers.instanceOf(TimeoutException.class)); final TestPrefetchableTextFilesFirehoseFactory factory = TestPrefetchableTextFilesFirehoseFactory.withSleepMillis(TEST_DIR, 1000); try (Firehose firehose = factory.connect(parser, createFirehoseTmpDir("testTimeout"))) { while (firehose.hasMore()) { firehose.nextRow(); } } }
@Test public void testWithoutCacheAndFetchAgainstConnectionReset() throws IOException { final TestPrefetchableTextFilesFirehoseFactory factory = TestPrefetchableTextFilesFirehoseFactory.withConnectionResets(TEST_DIR, 0, 0, 2); final List<Row> rows = new ArrayList<>(); final File firehoseTmpDir = createFirehoseTmpDir("testWithoutCacheAndFetch"); try (Firehose firehose = factory.connect(parser, firehoseTmpDir)) { while (firehose.hasMore()) { rows.add(firehose.nextRow()); } } Assert.assertEquals(0, factory.getCacheManager().getTotalCachedBytes()); assertResult(rows); assertNumRemainingCacheFiles(firehoseTmpDir, 0); }
@Test public void testMaxRetry() throws IOException { expectedException.expect(RuntimeException.class); expectedException.expectCause(CoreMatchers.instanceOf(ExecutionException.class)); expectedException.expectMessage("Exception for retry test"); final TestPrefetchableTextFilesFirehoseFactory factory = TestPrefetchableTextFilesFirehoseFactory.withOpenExceptions(TEST_DIR, 5); try (Firehose firehose = factory.connect(parser, createFirehoseTmpDir("testMaxRetry"))) { while (firehose.hasMore()) { firehose.nextRow(); } } }
@Test public void testWithoutCacheAndFetch() throws IOException { final TestPrefetchableTextFilesFirehoseFactory factory = TestPrefetchableTextFilesFirehoseFactory.with(TEST_DIR, 0, 0); final List<Row> rows = new ArrayList<>(); final File firehoseTmpDir = createFirehoseTmpDir("testWithoutCacheAndFetch"); try (Firehose firehose = factory.connect(parser, firehoseTmpDir)) { while (firehose.hasMore()) { rows.add(firehose.nextRow()); } } Assert.assertEquals(0, factory.getCacheManager().getTotalCachedBytes()); assertResult(rows); assertNumRemainingCacheFiles(firehoseTmpDir, 0); }
@Test public void testWithoutCache() throws IOException { final TestPrefetchableTextFilesFirehoseFactory factory = TestPrefetchableTextFilesFirehoseFactory.with(TEST_DIR, 0, 2048); final List<Row> rows = new ArrayList<>(); final File firehoseTmpDir = createFirehoseTmpDir("testWithoutCache"); try (Firehose firehose = factory.connect(parser, firehoseTmpDir)) { while (firehose.hasMore()) { rows.add(firehose.nextRow()); } } Assert.assertEquals(0, factory.getCacheManager().getTotalCachedBytes()); assertResult(rows); assertNumRemainingCacheFiles(firehoseTmpDir, 0); }
@Test public void testCombiningfirehose() throws IOException { List<InputRow> list1 = Arrays.asList(makeRow(1, 1), makeRow(2, 2)); List<InputRow> list2 = Arrays.asList(makeRow(3, 3), makeRow(4, 4), makeRow(5, 5)); FirehoseFactory combiningFactory = new CombiningFirehoseFactory( Arrays.asList( new ListFirehoseFactory(list1), new ListFirehoseFactory(list2) ) ); final Firehose firehose = combiningFactory.connect(null, null); for (int i = 1; i < 6; i++) { Assert.assertTrue(firehose.hasMore()); final InputRow inputRow = firehose.nextRow(); Assert.assertEquals(i, inputRow.getTimestampFromEpoch()); Assert.assertEquals(i, inputRow.getMetric("test").floatValue(), 0); } Assert.assertFalse(firehose.hasMore()); }
@Test public void testReconnectWithCache() throws IOException { final TestPrefetchableTextFilesFirehoseFactory factory = TestPrefetchableTextFilesFirehoseFactory.with(TEST_DIR, 2048, 0); final File firehoseTmpDir = createFirehoseTmpDir("testReconnectWithCache"); for (int i = 0; i < 5; i++) { final List<Row> rows = new ArrayList<>(); try (Firehose firehose = factory.connect(parser, firehoseTmpDir)) { if (i > 0) { Assert.assertEquals(FILE_SIZE * 2, factory.getCacheManager().getTotalCachedBytes()); } while (firehose.hasMore()) { rows.add(firehose.nextRow()); } } assertResult(rows); assertNumRemainingCacheFiles(firehoseTmpDir, 2); } }
@Test public void testReconnectWithCacheAndPrefetch() throws IOException { final TestPrefetchableTextFilesFirehoseFactory factory = TestPrefetchableTextFilesFirehoseFactory.of(TEST_DIR); final File firehoseTmpDir = createFirehoseTmpDir("testReconnectWithCacheAndPrefetch"); for (int i = 0; i < 5; i++) { final List<Row> rows = new ArrayList<>(); try (Firehose firehose = factory.connect(parser, firehoseTmpDir)) { if (i > 0) { Assert.assertEquals(FILE_SIZE * 2, factory.getCacheManager().getTotalCachedBytes()); } while (firehose.hasMore()) { rows.add(firehose.nextRow()); } } assertResult(rows); assertNumRemainingCacheFiles(firehoseTmpDir, 2); } }