@Override public Sink getSink(long timestamp) { if (!rejectionPolicy.accept(timestamp)) { return null; } final long truncatedTime = segmentGranularity.truncate(timestamp); Sink retVal = sinks.get(truncatedTime); if (retVal == null) { final Interval sinkInterval = new Interval( new DateTime(truncatedTime), segmentGranularity.increment(new DateTime(truncatedTime)) ); retVal = new Sink(sinkInterval, schema, versioningPolicy.getVersion(sinkInterval)); try { segmentAnnouncer.announceSegment(retVal.getSegment()); sinks.put(truncatedTime, retVal); sinkTimeline.add(retVal.getInterval(), retVal.getVersion(), new SingleElementPartitionChunk<Sink>(retVal)); } catch (IOException e) { log.makeAlert(e, "Failed to announce new segment[%s]", schema.getDataSource()) .addData("interval", retVal.getInterval()) .emit(); } } return retVal; }
@Override public int add(InputRow row, Supplier<Committer> committerSupplier) throws IndexSizeExceededException { Sink sink = getSink(row.getTimestampFromEpoch()); if (sink == null) { return -1; } final int numRows = sink.add(row, false); if (!sink.canAppendRow()) { persist(committerSupplier.get()); } return numRows; }
@Override public void persist(final Runnable commitRunnable) { final List<Pair<FireHydrant, Interval>> indexesToPersist = Lists.newArrayList(); for (Sink sink : sinks.values()) { if (sink.swappable()) { indexesToPersist.add(Pair.of(sink.swap(), sink.getInterval())); } } log.info("Submitting persist runnable for dataSource[%s]", schema.getDataSource()); persistExecutor.execute( new ThreadRenamingRunnable(String.format("%s-incremental-persist", schema.getDataSource())) { @Override public void doRun() { for (Pair<FireHydrant, Interval> pair : indexesToPersist) { metrics.incrementRowOutputCount(persistHydrant(pair.lhs, schema, pair.rhs)); } commitRunnable.run(); } } ); }
private void addSink(final Sink sink) { sinks.put(sink.getInterval().getStartMillis(), sink); metrics.setSinkCount(sinks.size()); sinkTimeline.add( sink.getInterval(), sink.getVersion(), new SingleElementPartitionChunk<Sink>(sink) ); try { segmentAnnouncer.announceSegment(sink.getSegment()); } catch (IOException e) { log.makeAlert(e, "Failed to announce new segment[%s]", schema.getDataSource()) .addData("interval", sink.getInterval()) .emit(); } }
numPersistedRows += sink.getNumRowsInMemory(); final int limit = sink.isWritable() ? hydrants.size() - 1 : hydrants.size(); if (sink.swappable()) { indexesToPersist.add(Pair.of(sink.swap(), identifier));
@Override public String apply(Sink input) { return input.getSegment().getIdentifier(); } }
private void spillIfSwappable() { if(theSink.swappable()) { final FireHydrant indexToPersist = theSink.swap(); final int rowsToPersist = indexToPersist.getIndex().size(); final File dirToPersist = getSpillDir(indexToPersist.getCount()); log.info("Spilling index[%d] with rows[%d] to: %s", indexToPersist.getCount(), rowsToPersist, dirToPersist); try { IndexMerger.persist( indexToPersist.getIndex(), dirToPersist ); indexToPersist.swapSegment(null); metrics.incrementRowOutputCount(rowsToPersist); spilled.add(dirToPersist); } catch(Exception e) { log.warn(e, "Failed to spill index[%d]", indexToPersist.getCount()); throw Throwables.propagate(e); } } }
Preconditions.checkState(!theSink.swappable(), "All data must be persisted before fininshing the job!"); final DataSegment segmentToUpload = theSink.getSegment() .withDimensions(ImmutableList.copyOf(mappedSegment.getAvailableDimensions())) .withBinaryVersion(SegmentUtils.getVersionFromDir(fileToUpload));
@Override public void doRun() final Interval interval = sink.getInterval(); sink.getSegment().withDimensions(Lists.newArrayList(index.getAvailableDimensions())) );
final Sink theSink = new Sink(interval, schema, version); final File persistDir = new File(tmpSegmentDir, theSink.getSegment().getIdentifier());
final int sinkRowsInMemoryBeforeAdd = sink.getNumRowsInMemory(); final int sinkRowsInMemoryAfterAdd; sinkRowsInMemoryAfterAdd = sink.add(row, !allowIncrementalPersists); if (!sink.canAppendRow() || System.currentTimeMillis() > nextFlush || rowsCurrentlyInMemory.get() >= tuningConfig.getMaxRowsInMemory()) { return new AppenderatorAddResult(identifier, sink.getNumRows(), isPersistRequired);
new ThreadRenamingRunnable(threadName) final Interval interval = sink.getInterval(); Stopwatch mergeStopwatch = null; new SegmentDescriptor(sink.getInterval(), sink.getVersion(), config.getShardSpec().getPartitionNum()), mergeExecutor, new Runnable()
private Sink getSink(long timestamp) { if (theSink.getInterval().contains(timestamp)) { return theSink; } else { return null; } }
if (sink.isWritable()) { throw new ISE("WTF?! Expected sink to be no longer writable before mergeAndPush. Segment[%s].", identifier); sink.getSegment().withDimensions(IndexMerger.getMergedDimensionsFromQueryableIndexes(indexes)), useUniquePath ),
continue; final Sink currSink = new Sink( sinkInterval, schema,
int currCount = sink.add(inputRow); if (currCount >= config.getMaxRowsInMemory() || System.currentTimeMillis() > nextFlush) { plumber.persist(firehose.commit());
/** * If currIndex is A, creates a new index B, sets currIndex to B and returns A. * * @return the current index after swapping in a new one */ public FireHydrant swap() { return makeNewCurrIndex(interval.getStartMillis(), schema); }
/** * Unannounces a given sink and removes all local references to it. */ protected void abandonSegment(final long truncatedTime, final Sink sink) { try { segmentAnnouncer.unannounceSegment(sink.getSegment()); FileUtils.deleteDirectory(computePersistDir(schema, sink.getInterval())); log.info("Removing sinkKey %d for segment %s", truncatedTime, sink.getSegment().getIdentifier()); sinks.remove(truncatedTime); sinkTimeline.remove( sink.getInterval(), sink.getVersion(), new SingleElementPartitionChunk<>(sink) ); synchronized (handoffCondition) { handoffCondition.notifyAll(); } } catch (IOException e) { log.makeAlert(e, "Unable to abandon old segment for dataSource[%s]", schema.getDataSource()) .addData("interval", sink.getInterval()) .emit(); } }
@Override public String apply(Sink input) { return input.getSegment().getIdentifier(); } }