@Override public TimestampedLong aggregate(TimestampedLong oldValue, TimestampedLong toAdd) { if (oldValue == null && toAdd == null) { return new TimestampedLong(0L); } else if (oldValue == null) { return toAdd; } else if (toAdd == null) { return oldValue; } return new TimestampedLong(oldValue.getValue() + toAdd.getValue()); }
@SuppressWarnings("unchecked") public TimedBuffer(final TimeUnit binPrecision, final int numBins, final EntityAccess<T> accessor) { this.binPrecision = binPrecision; this.numBins = numBins + 1; this.bins = new EntitySum[this.numBins]; for (int i = 0; i < this.numBins; i++) { this.bins[i] = new EntitySum<>(binPrecision, numBins, accessor); } this.entityAccess = accessor; }
@Override public TimedCountSize aggregate(final TimedCountSize oldValue, final TimedCountSize toAdd) { if (oldValue == null && toAdd == null) { return new TimedCountSize(0L, 0L); } else if (oldValue == null) { return toAdd; } else if (toAdd == null) { return oldValue; } return new TimedCountSize(oldValue.getCount() + toAdd.getCount(), oldValue.getSize() + toAdd.getSize()); }
final TimestampedLong sum = timedBuffer.getAggregateValue(timePeriodMillis); if (sum != null && sum.getValue() >= maxRateValue) { if (logger.isDebugEnabled()) { logger.debug("current sum for throttle is {} at time {}, so not allowing rate of {} through", new Object[]{sum.getValue(), sum.getTimestamp(), value}); new Object[]{sum == null ? 0 : sum.getValue(), sum == null ? 0 : sum.getTimestamp(), value}); final long transferred = timedBuffer.add(new TimestampedLong(value)).getValue(); if (transferred > maxRateValue) { final long amountOver = transferred - maxRateValue;
@Override public void addEvents(final Map<ProvenanceEventRecord, StorageSummary> events) { eventsIndexed.add(new TimestampedLong((long) events.size())); for (final Map.Entry<ProvenanceEventRecord, StorageSummary> entry : events.entrySet()) { addEvent(entry.getKey(), entry.getValue()); } }
private void incrementAndReportStats() { final long fiveMinutesAgo = System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(5); final TimestampedLong nanosLastFive = queuePauseNanos.getAggregateValue(fiveMinutesAgo); if (nanosLastFive == null) { return; } final TimestampedLong eventsLast5 = eventsIndexed.getAggregateValue(fiveMinutesAgo); if (eventsLast5 == null) { return; } final long numEventsLast5 = eventsLast5.getValue(); final long millis = TimeUnit.NANOSECONDS.toMillis(nanosLastFive.getValue()); logger.debug("In the last 5 minutes, have spent {} CPU-millis waiting to enqueue events for indexing and have indexed {} events ({} since NiFi started)", millis, numEventsLast5, eventCount.get()); }
final TimedCountSize countSize = updateCounts.getAggregateValue(System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES)); logger.info("Successfully Rolled over Provenance Event file containing {} records. In the past 5 minutes, " + "{} events have been written to the Provenance Repository, totaling {}", recordsWritten, countSize.getCount(), FormatUtils.formatDataSize(countSize.getSize()));
this.updateCounts.add(new TimedCountSize(recordsWritten, bytesWritten)); } catch (final Throwable t) {
@Override public TimestampedLong createNew() { return new TimestampedLong(0L); }
@Override public TimedCountSize createNew() { return new TimedCountSize(0L, 0L); }
public T getAggregateValue(final long sinceEpochMillis) { final int startBinIdx = (int) (binPrecision.convert(sinceEpochMillis, TimeUnit.MILLISECONDS) % numBins); T total = null; for (int i = 0; i < numBins; i++) { int binIdx = (startBinIdx + i) % numBins; final EntitySum<T> bin = bins[binIdx]; if (!bin.isExpired()) { total = entityAccess.aggregate(total, bin.getValue()); } } return total; }
public S addOrReset(final S event) { // entityAccess.getTimestamp(curValue) represents the time at which the current value // was last updated. If the last value is less than current time - 1 binPrecision, then it // means that we've rolled over and need to reset the value. final long maxExpectedTimePeriod = System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(1, binPrecision); final S curValue = ref.get(); if (entityAccess.getTimestamp(curValue) < maxExpectedTimePeriod) { ref.compareAndSet(curValue, entityAccess.createNew()); } return add(event); } }
@Override public long getTimestamp(TimestampedLong entity) { return entity == null ? 0L : entity.getTimestamp(); } }
public Throttle(final int timePeriod, final TimeUnit unit, final ComponentLog logger) { this.timePeriodMillis = TimeUnit.MILLISECONDS.convert(timePeriod, unit); this.timedBuffer = new TimedBuffer<>(unit, timePeriod, new LongEntityAccess()); this.logger = logger; }
@Override public long getTimestamp(final TimedCountSize entity) { return entity == null ? 0L : entity.getTimestamp(); } }
private S add(final S event) { S newValue; S value; do { value = ref.get(); newValue = entityAccess.aggregate(value, event); } while (!ref.compareAndSet(value, newValue)); return newValue; }
public T add(final T entity) { final int binIdx = (int) (binPrecision.convert(System.currentTimeMillis(), TimeUnit.MILLISECONDS) % numBins); final EntitySum<T> sum = bins[binIdx]; return sum.addOrReset(entity); }
public boolean isExpired() { // entityAccess.getTimestamp(curValue) represents the time at which the current value // was last updated. If the last value is less than current time - 1 binPrecision, then it // means that we've rolled over and need to reset the value. final long maxExpectedTimePeriod = System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(numConfiguredBins, binPrecision); final S curValue = ref.get(); return (entityAccess.getTimestamp(curValue) < maxExpectedTimePeriod); }
getWriteTimes().add(new TimestampedLong(writeNanos)); getSerializeTimes().add(new TimestampedLong(serializeNanos)); getEncryptTimes().add(new TimestampedLong(encryptNanos)); getLockTimes().add(new TimestampedLong(lockNanos)); getBytesWrittenBuffer().add(new TimestampedLong(endBytes - startBytes));
private void printStats() { final long sixtySecondsAgo = System.currentTimeMillis() - 60000L; final Long writeNanosLast60 = getWriteTimes().getAggregateValue(sixtySecondsAgo).getValue(); final Long lockNanosLast60 = getLockTimes().getAggregateValue(sixtySecondsAgo).getValue(); final Long serializeNanosLast60 = getSerializeTimes().getAggregateValue(sixtySecondsAgo).getValue(); final Long encryptNanosLast60 = getEncryptTimes().getAggregateValue(sixtySecondsAgo).getValue(); final Long bytesWrittenLast60 = getBytesWrittenBuffer().getAggregateValue(sixtySecondsAgo).getValue(); logger.debug("In the last 60 seconds, have spent {} millis writing to file ({} MB), {} millis waiting on synchronize block, {} millis serializing events, {} millis encrypting events", TimeUnit.NANOSECONDS.toMillis(writeNanosLast60), bytesWrittenLast60 / 1024 / 1024, TimeUnit.NANOSECONDS.toMillis(lockNanosLast60), TimeUnit.NANOSECONDS.toMillis(serializeNanosLast60), TimeUnit.NANOSECONDS.toMillis(encryptNanosLast60)); }