@Override public TimestampedLong aggregate(TimestampedLong oldValue, TimestampedLong toAdd) { if (oldValue == null && toAdd == null) { return new TimestampedLong(0L); } else if (oldValue == null) { return toAdd; } else if (toAdd == null) { return oldValue; } return new TimestampedLong(oldValue.getValue() + toAdd.getValue()); }
@Override public long getTimestamp(TimestampedLong entity) { return entity == null ? 0L : entity.getTimestamp(); } }
if (sum != null && sum.getValue() >= maxRateValue) { if (logger.isDebugEnabled()) { logger.debug("current sum for throttle is {} at time {}, so not allowing rate of {} through", new Object[]{sum.getValue(), sum.getTimestamp(), value}); new Object[]{sum == null ? 0 : sum.getValue(), sum == null ? 0 : sum.getTimestamp(), value}); final long transferred = timedBuffer.add(new TimestampedLong(value)).getValue(); if (transferred > maxRateValue) { final long amountOver = transferred - maxRateValue;
@Override public TimestampedLong createNew() { return new TimestampedLong(0L); }
private void incrementAndReportStats() { final long fiveMinutesAgo = System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(5); final TimestampedLong nanosLastFive = queuePauseNanos.getAggregateValue(fiveMinutesAgo); if (nanosLastFive == null) { return; } final TimestampedLong eventsLast5 = eventsIndexed.getAggregateValue(fiveMinutesAgo); if (eventsLast5 == null) { return; } final long numEventsLast5 = eventsLast5.getValue(); final long millis = TimeUnit.NANOSECONDS.toMillis(nanosLastFive.getValue()); logger.debug("In the last 5 minutes, have spent {} CPU-millis waiting to enqueue events for indexing and have indexed {} events ({} since NiFi started)", millis, numEventsLast5, eventCount.get()); }
if (sum != null && sum.getValue() >= maxRateValue) { if (logger.isDebugEnabled()) { logger.debug("current sum for throttle is {} at time {}, so not allowing rate of {} through", new Object[]{sum.getValue(), sum.getTimestamp(), value}); new Object[]{sum == null ? 0 : sum.getValue(), sum == null ? 0 : sum.getTimestamp(), value}); final long transferred = timedBuffer.add(new TimestampedLong(value)).getValue(); if (transferred > maxRateValue) { final long amountOver = transferred - maxRateValue;
@Override public void addEvents(final Map<ProvenanceEventRecord, StorageSummary> events) { eventsIndexed.add(new TimestampedLong((long) events.size())); for (final Map.Entry<ProvenanceEventRecord, StorageSummary> entry : events.entrySet()) { addEvent(entry.getKey(), entry.getValue()); } }
private void printStats() { final long sixtySecondsAgo = System.currentTimeMillis() - 60000L; final Long writeNanosLast60 = getWriteTimes().getAggregateValue(sixtySecondsAgo).getValue(); final Long lockNanosLast60 = getLockTimes().getAggregateValue(sixtySecondsAgo).getValue(); final Long serializeNanosLast60 = getSerializeTimes().getAggregateValue(sixtySecondsAgo).getValue(); final Long encryptNanosLast60 = getEncryptTimes().getAggregateValue(sixtySecondsAgo).getValue(); final Long bytesWrittenLast60 = getBytesWrittenBuffer().getAggregateValue(sixtySecondsAgo).getValue(); logger.debug("In the last 60 seconds, have spent {} millis writing to file ({} MB), {} millis waiting on synchronize block, {} millis serializing events, {} millis encrypting events", TimeUnit.NANOSECONDS.toMillis(writeNanosLast60), bytesWrittenLast60 / 1024 / 1024, TimeUnit.NANOSECONDS.toMillis(lockNanosLast60), TimeUnit.NANOSECONDS.toMillis(serializeNanosLast60), TimeUnit.NANOSECONDS.toMillis(encryptNanosLast60)); }
backpressurePauseMillis.add(new TimestampedLong(stopTheWorldNanos)); final TimestampedLong pauseNanosLastFiveMinutes = backpressurePauseMillis.getAggregateValue(System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES)); logger.info("Provenance Repository has now caught up with rolling over journal files. Current number of " + "journal files to be rolled over is {}. Provenance Repository Back Pressure paused Session commits for {} ({} total in the last 5 minutes).", journalFileCount, FormatUtils.formatNanos(stopTheWorldNanos, true), FormatUtils.formatNanos(pauseNanosLastFiveMinutes.getValue(), true));
getWriteTimes().add(new TimestampedLong(writeNanos)); getSerializeTimes().add(new TimestampedLong(serializeNanos)); getEncryptTimes().add(new TimestampedLong(encryptNanos)); getLockTimes().add(new TimestampedLong(lockNanos)); getBytesWrittenBuffer().add(new TimestampedLong(endBytes - startBytes));
@Override public long getTimestamp(TimestampedLong entity) { return entity == null ? 0L : entity.getTimestamp(); } }
@Override public TimestampedLong aggregate(TimestampedLong oldValue, TimestampedLong toAdd) { if (oldValue == null && toAdd == null) { return new TimestampedLong(0L); } else if (oldValue == null) { return toAdd; } else if (toAdd == null) { return oldValue; } return new TimestampedLong(oldValue.getValue() + toAdd.getValue()); }
@Override public TimestampedLong createNew() { return new TimestampedLong(0L); }
backpressurePauseMillis.add(new TimestampedLong(stopTheWorldNanos)); final TimestampedLong pauseNanosLastFiveMinutes = backpressurePauseMillis.getAggregateValue(System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES)); logger.info("Provenance Repository has now caught up with rolling over journal files. Current number of " + "journal files to be rolled over is {}. Provenance Repository Back Pressure paused Session commits for {} ({} total in the last 5 minutes).", journalFileCount, FormatUtils.formatNanos(stopTheWorldNanos, true), FormatUtils.formatNanos(pauseNanosLastFiveMinutes.getValue(), true));