@Override public void addEvents(final Map<ProvenanceEventRecord, StorageSummary> events) { eventsIndexed.add(new TimestampedLong((long) events.size())); for (final Map.Entry<ProvenanceEventRecord, StorageSummary> entry : events.entrySet()) { addEvent(entry.getKey(), entry.getValue()); } }
private void incrementAndReportStats() { final long fiveMinutesAgo = System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(5); final TimestampedLong nanosLastFive = queuePauseNanos.getAggregateValue(fiveMinutesAgo); if (nanosLastFive == null) { return; } final TimestampedLong eventsLast5 = eventsIndexed.getAggregateValue(fiveMinutesAgo); if (eventsLast5 == null) { return; } final long numEventsLast5 = eventsLast5.getValue(); final long millis = TimeUnit.NANOSECONDS.toMillis(nanosLastFive.getValue()); logger.debug("In the last 5 minutes, have spent {} CPU-millis waiting to enqueue events for indexing and have indexed {} events ({} since NiFi started)", millis, numEventsLast5, eventCount.get()); }
public Throttle(final int timePeriod, final TimeUnit unit, final ComponentLog logger) { this.timePeriodMillis = TimeUnit.MILLISECONDS.convert(timePeriod, unit); this.timedBuffer = new TimedBuffer<>(unit, timePeriod, new LongEntityAccess()); this.logger = logger; }
backpressurePauseMillis.add(new TimestampedLong(stopTheWorldNanos)); final TimestampedLong pauseNanosLastFiveMinutes = backpressurePauseMillis.getAggregateValue(System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES)); logger.info("Provenance Repository has now caught up with rolling over journal files. Current number of " + "journal files to be rolled over is {}. Provenance Repository Back Pressure paused Session commits for {} ({} total in the last 5 minutes).",
final TimestampedLong sum = timedBuffer.getAggregateValue(timePeriodMillis); if (sum != null && sum.getValue() >= maxRateValue) { if (logger.isDebugEnabled()) { final long transferred = timedBuffer.add(new TimestampedLong(value)).getValue(); if (transferred > maxRateValue) { final long amountOver = transferred - maxRateValue;
backpressurePauseMillis.add(new TimestampedLong(stopTheWorldNanos)); final TimestampedLong pauseNanosLastFiveMinutes = backpressurePauseMillis.getAggregateValue(System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES)); logger.info("Provenance Repository has now caught up with rolling over journal files. Current number of " + "journal files to be rolled over is {}. Provenance Repository Back Pressure paused Session commits for {} ({} total in the last 5 minutes).",
this.updateCounts.add(new TimedCountSize(recordsWritten, bytesWritten)); } catch (final Throwable t) {
private void printStats() { final long sixtySecondsAgo = System.currentTimeMillis() - 60000L; final Long writeNanosLast60 = getWriteTimes().getAggregateValue(sixtySecondsAgo).getValue(); final Long lockNanosLast60 = getLockTimes().getAggregateValue(sixtySecondsAgo).getValue(); final Long serializeNanosLast60 = getSerializeTimes().getAggregateValue(sixtySecondsAgo).getValue(); final Long encryptNanosLast60 = getEncryptTimes().getAggregateValue(sixtySecondsAgo).getValue(); final Long bytesWrittenLast60 = getBytesWrittenBuffer().getAggregateValue(sixtySecondsAgo).getValue(); logger.debug("In the last 60 seconds, have spent {} millis writing to file ({} MB), {} millis waiting on synchronize block, {} millis serializing events, {} millis encrypting events", TimeUnit.NANOSECONDS.toMillis(writeNanosLast60), bytesWrittenLast60 / 1024 / 1024, TimeUnit.NANOSECONDS.toMillis(lockNanosLast60), TimeUnit.NANOSECONDS.toMillis(serializeNanosLast60), TimeUnit.NANOSECONDS.toMillis(encryptNanosLast60)); }
public Throttle(final int timePeriod, final TimeUnit unit, final ComponentLog logger) { this.timePeriodMillis = TimeUnit.MILLISECONDS.convert(timePeriod, unit); this.timedBuffer = new TimedBuffer<>(unit, timePeriod, new LongEntityAccess()); this.logger = logger; }
final TimestampedLong sum = timedBuffer.getAggregateValue(timePeriodMillis); if (sum != null && sum.getValue() >= maxRateValue) { if (logger.isDebugEnabled()) { final long transferred = timedBuffer.add(new TimestampedLong(value)).getValue(); if (transferred > maxRateValue) { final long amountOver = transferred - maxRateValue;
final TimedCountSize countSize = updateCounts.getAggregateValue(System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES)); logger.info("Successfully Rolled over Provenance Event file containing {} records. In the past 5 minutes, " + "{} events have been written to the Provenance Repository, totaling {}",
this.updateCounts.add(new TimedCountSize(recordsWritten, bytesWritten)); } catch (final Throwable t) {
final TimedCountSize countSize = updateCounts.getAggregateValue(System.currentTimeMillis() - TimeUnit.MILLISECONDS.convert(5, TimeUnit.MINUTES)); logger.info("Successfully Rolled over Provenance Event file containing {} records. In the past 5 minutes, " + "{} events have been written to the Provenance Repository, totaling {}",