@Override public void persist(final Runnable commitRunnable) { final List<Pair<FireHydrant, Interval>> indexesToPersist = Lists.newArrayList(); for (Sink sink : sinks.values()) { if (sink.swappable()) { indexesToPersist.add(Pair.of(sink.swap(), sink.getInterval())); } } log.info("Submitting persist runnable for dataSource[%s]", schema.getDataSource()); persistExecutor.execute( new ThreadRenamingRunnable(String.format("%s-incremental-persist", schema.getDataSource())) { @Override public void doRun() { for (Pair<FireHydrant, Interval> pair : indexesToPersist) { metrics.incrementRowOutputCount(persistHydrant(pair.lhs, schema, pair.rhs)); } commitRunnable.run(); } } ); }
for (Sink sink : sinks.values()) { if (sink.swappable()) { indexesToPersist.add(Pair.of(sink.swap(), sink.getInterval()));
indexesToPersist.add(Pair.of(sink.swap(), identifier));
private void spillIfSwappable() { if(theSink.swappable()) { final FireHydrant indexToPersist = theSink.swap(); final int rowsToPersist = indexToPersist.getIndex().size(); final File dirToPersist = getSpillDir(indexToPersist.getCount()); log.info("Spilling index[%d] with rows[%d] to: %s", indexToPersist.getCount(), rowsToPersist, dirToPersist); try { IndexMerger.persist( indexToPersist.getIndex(), dirToPersist ); indexToPersist.swapSegment(null); metrics.incrementRowOutputCount(rowsToPersist); spilled.add(dirToPersist); } catch(Exception e) { log.warn(e, "Failed to spill index[%d]", indexToPersist.getCount()); throw Throwables.propagate(e); } } }
private void spillIfSwappable() { if (theSink.swappable()) { final FireHydrant indexToPersist = theSink.swap(); final int rowsToPersist = indexToPersist.getIndex().size(); final File dirToPersist = getSpillDir(indexToPersist.getCount()); log.info("Spilling index[%d] with rows[%d] to: %s", indexToPersist.getCount(), rowsToPersist, dirToPersist); try { indexMergerV9.persist( indexToPersist.getIndex(), dirToPersist, config.getIndexSpec(), config.getSegmentWriteOutMediumFactory() ); indexToPersist.swapSegment(null); metrics.incrementRowOutputCount(rowsToPersist); spilled.add(dirToPersist); } catch (Exception e) { log.warn(e, "Failed to spill index[%d]", indexToPersist.getCount()); throw Throwables.propagate(e); } } }