/** * Initialization method that connects up the fire hose. If this method returns successfully it should be safe to * call hasMore() on the returned Firehose (which might subsequently block). * <p/> * If this method returns null, then any attempt to call hasMore(), nextRow(), commit() and close() on the return * value will throw a surprising NPE. Throwing IOException on connection failure or runtime exception on * invalid configuration is preferred over returning null. * * @param parser an input row parser */ @Deprecated default Firehose connect(T parser) throws IOException, ParseException { return connect(parser, null); }
/** * Initialization method that connects up the fire hose. If this method returns successfully it should be safe to * call hasMore() on the returned Firehose (which might subsequently block). * <p/> * If this method returns null, then any attempt to call hasMore(), nextRow(), commit() and close() on the return * value will throw a surprising NPE. Throwing IOException on connection failure or runtime exception on * invalid configuration is preferred over returning null. * <p/> * Some fire hoses like {@link PrefetchableTextFilesFirehoseFactory} may use a temporary * directory to cache data in it. * * @param parser an input row parser * @param temporaryDirectory a directory where temporary files are stored */ default Firehose connect(T parser, @Nullable File temporaryDirectory) throws IOException, ParseException { return connect(parser); }
private Firehose delegateFirehose = delegate.connect(parser, temporaryDirectory);
@Override public Firehose connect(InputRowParser parser, File temporaryDirectory) throws IOException { return new PredicateFirehose( delegate.connect(parser, temporaryDirectory), new Predicate<InputRow>() { @Override public boolean apply(InputRow input) { return interval.contains(input.getTimestampFromEpoch()); } } ); }
public Firehose connect() throws IOException { return ioConfig.getFirehoseFactory().connect(dataSchema.getParser(), null); }
private void nextFirehose() { if (firehoseFactoryIterator.hasNext()) { try { if (currentFirehose != null) { currentFirehose.close(); } currentFirehose = firehoseFactoryIterator.next().connect(parser, temporaryDirectory); } catch (IOException e) { if (currentFirehose != null) { try { currentFirehose.close(); } catch (IOException e2) { log.error(e, "Unable to close currentFirehose!"); throw Throwables.propagate(e2); } } throw Throwables.propagate(e); } } }
TimedShutoffFirehose(InputRowParser parser, File temporaryDirectory) throws IOException { firehose = delegateFactory.connect(parser, temporaryDirectory); exec = Execs.scheduledSingleThreaded("timed-shutoff-firehose-%d"); exec.schedule( new Runnable() { @Override public void run() { log.info("Closing delegate firehose."); shutdown = true; try { firehose.close(); } catch (IOException e) { log.warn(e, "Failed to close delegate firehose, ignoring."); } } }, shutoffTime.getMillis() - System.currentTimeMillis(), TimeUnit.MILLISECONDS ); log.info("Firehose created, will shut down at: %s", shutoffTime); }
@Override public TaskStatus run(TaskToolbox toolbox) throws Exception { if (firehoseFactory != null) { log.info("Connecting firehose"); } try (Firehose firehose = firehoseFactory != null ? firehoseFactory.connect(null, null) : null) { log.info("Running noop task[%s]", getId()); log.info("Sleeping for %,d millis.", runTime); Thread.sleep(runTime); log.info("Woke up!"); return TaskStatus.success(getId()); } }
final Firehose firehose = firehoseFactory.connect(ingestionSchema.getDataSchema().getParser(), firehoseTempDir) ) {
final Appenderator appenderator = newAppenderator(fireDepartmentMetrics, toolbox, dataSchema, tuningConfig); final BatchAppenderatorDriver driver = newDriver(appenderator, toolbox, segmentAllocator); final Firehose firehose = firehoseFactory.connect(dataSchema.getParser(), firehoseTempDir) ) { driver.startJob();
@Test public void testCombiningfirehose() throws IOException { List<InputRow> list1 = Arrays.asList(makeRow(1, 1), makeRow(2, 2)); List<InputRow> list2 = Arrays.asList(makeRow(3, 3), makeRow(4, 4), makeRow(5, 5)); FirehoseFactory combiningFactory = new CombiningFirehoseFactory( Arrays.asList( new ListFirehoseFactory(list1), new ListFirehoseFactory(list2) ) ); final Firehose firehose = combiningFactory.connect(null, null); for (int i = 1; i < 6; i++) { Assert.assertTrue(firehose.hasMore()); final InputRow inputRow = firehose.nextRow(); Assert.assertEquals(i, inputRow.getTimestampFromEpoch()); Assert.assertEquals(i, inputRow.getMetric("test").floatValue(), 0); } Assert.assertFalse(firehose.hasMore()); }
); final BatchAppenderatorDriver driver = newDriver(appenderator, toolbox, segmentAllocator); final Firehose firehose = firehoseFactory.connect(dataSchema.getParser(), firehoseTempDir) ) { driver.startJob();
firehose = firehoseFactory.connect(spec.getDataSchema().getParser(), firehoseTempDir); committerSupplier = Committers.supplierFromFirehose(firehose);
firehose = firehoseFactory.connect(spec.getDataSchema().getParser(), firehoseTempDir); committerSupplier = Committers.supplierFromFirehose(firehose);
private Firehose delegateFirehose = delegate.connect(parser, temporaryDirectory);
/** * Initialization method that connects up the fire hose. If this method returns successfully it should be safe to * call hasMore() on the returned Firehose (which might subsequently block). * <p/> * If this method returns null, then any attempt to call hasMore(), nextRow(), commit() and close() on the return * value will throw a surprising NPE. Throwing IOException on connection failure or runtime exception on * invalid configuration is preferred over returning null. * * @param parser an input row parser */ @Deprecated default Firehose connect(T parser) throws IOException, ParseException { return connect(parser, null); }
/** * Initialization method that connects up the fire hose. If this method returns successfully it should be safe to * call hasMore() on the returned Firehose (which might subsequently block). * <p/> * If this method returns null, then any attempt to call hasMore(), nextRow(), commit() and close() on the return * value will throw a surprising NPE. Throwing IOException on connection failure or runtime exception on * invalid configuration is preferred over returning null. * <p/> * Some fire hoses like {@link PrefetchableTextFilesFirehoseFactory} may use a temporary * directory to cache data in it. * * @param parser an input row parser * @param temporaryDirectory a directory where temporary files are stored */ default Firehose connect(T parser, @Nullable File temporaryDirectory) throws IOException, ParseException { return connect(parser); }
@Override public Firehose connect(InputRowParser parser, File temporaryDirectory) throws IOException { return new PredicateFirehose( delegate.connect(parser, temporaryDirectory), new Predicate<InputRow>() { @Override public boolean apply(InputRow input) { return interval.contains(input.getTimestampFromEpoch()); } } ); }
public Firehose connect() throws IOException { return ioConfig.getFirehoseFactory().connect(dataSchema.getParser(), null); }
TimedShutoffFirehose(InputRowParser parser, File temporaryDirectory) throws IOException { firehose = delegateFactory.connect(parser, temporaryDirectory); exec = Execs.scheduledSingleThreaded("timed-shutoff-firehose-%d"); exec.schedule( new Runnable() { @Override public void run() { log.info("Closing delegate firehose."); shutdown = true; try { firehose.close(); } catch (IOException e) { log.warn(e, "Failed to close delegate firehose, ignoring."); } } }, shutoffTime.getMillis() - System.currentTimeMillis(), TimeUnit.MILLISECONDS ); log.info("Firehose created, will shut down at: %s", shutoffTime); }