public JobDag(@NonNull final ISource<K, R> source, @NonNull final JobSubDag sinkDag, @NonNull final IMetadataManager<V> metadataManager, @NonNull final IWorkUnitCalculator<T, R, K, V> workUnitCalculator, @NotEmpty final String jobName, @NotEmpty final String dataFeedName, @NonNull final JobMetrics jobMetrics, @NonNull final Reporters reporters) { this.source = source; this.sinkDag = sinkDag; this.metadataManager = metadataManager; this.workUnitCalculator = workUnitCalculator; this.jobName = jobName; this.dataFeedName = dataFeedName; this.reporters = reporters; this.postJobDagActions = new JobDagActions(this.reporters, dataFeedName); this.jobMetrics = jobMetrics; this.dataFeedMetrics = new DataFeedMetrics(this.jobName, Collections.singletonMap(DataFeedMetrics.DATA_FEED_NAME, this.dataFeedName)); }
private void computePartitionMetrics(@NonNull final TopicPartition topicPartition, final long leaderOffset, @NonNull final KafkaRunState nextRunState, @NonNull final DataFeedMetrics topicMetrics, @NonNull final Long inputCount, @NonNull final MessageCounters counter) { final Optional<Long> nextPartitionState = nextRunState.getPartitionOffset(topicPartition.partition()); final Map<String, String> tags = new HashMap<>(); tags.put(PARTITION_TAG, Integer.toString(topicPartition.partition())); if (nextPartitionState.isPresent()) { final long current = nextPartitionState.get(); counter.add(leaderOffset, current, inputCount); topicMetrics.createLongMetric(DataFeedMetricNames.AVAILABLE_ROWCOUNT, leaderOffset, tags); topicMetrics.createLongMetric(DataFeedMetricNames.CURRENT_STATUS, current, tags); topicMetrics.createLongMetric(DataFeedMetricNames.INPUT_ROWCOUNT, inputCount, tags); } else { topicMetrics.createLongMetric(DataFeedMetricNames.AVAILABLE_ROWCOUNT, leaderOffset, tags); topicMetrics.createLongMetric(DataFeedMetricNames.CURRENT_STATUS, leaderOffset, tags); topicMetrics.createLongMetric(DataFeedMetricNames.INPUT_ROWCOUNT, 0, tags); } }
@Override public boolean execute(final boolean success) { if (success) { this.reporters.getReporters().forEach(reporter -> { this.jobMetrics.gaugeAll(reporter); this.dataFeedMetrics.gaugeAll(reporter); }); } else { log.warn("No metrics produced or actions being executed on reporter because errors were encountered"); } return success; }
); topicMetrics.createLongMetric(DataFeedMetricNames.ROWCOUNT_BEHIND, counter.getTotalAvailable() - counter.getTotalCurrent(), totalTags); topicMetrics.createLongMetric(DataFeedMetricNames.INPUT_ROWCOUNT, counter.getTotalInput(), totalTags); if (this.chargebackCalculator.isPresent()) { this.chargebackCalculator.get().addCost( this.topicMetrics.get().getBaseTags().get(DataFeedMetrics.DATA_FEED_NAME), ChargebackMetricType.ROW_COUNT, counter.getTotalInput());
final ReporterAction reporterAction = new ReporterAction(this.reporters, this.jobMetrics, this.dataFeedMetrics); this.addAction(reporterAction); final TimerMetric timerMetric = this.dataFeedMetrics.createTimerMetric( DataFeedMetricNames.TOTAL_LATENCY_MS, new HashMap<>(), Optional.absent()); try { String.valueOf(TimeUnit.SECONDS.toMillis(timerMetric.getStartTime().getEpochSecond()))); this.dataFeedMetrics.createLongMetric(DataFeedMetricNames.RESULT, successful.get() ? DataFeedMetricNames.RESULT_SUCCESS : DataFeedMetricNames.RESULT_FAILURE, Collections.emptyMap());
@Test public void testCreateTopicTimerMetric() { Metric topicMetric1 = dataFeedMetrics.createTimerMetric(DataFeedMetricNames.RUNTIME); assertEquals("runtime", topicMetric1.getMetricName()); assertTrue(topicMetric1 instanceof TimerMetric); assertTrue(topicMetric1.getTags().containsKey("job")); assertEquals(topicMetric1.getTags().get("job"), "test-job"); assertTrue(topicMetric1.getTags().containsKey("datafeed")); assertEquals(topicMetric1.getTags().get("datafeed"), "test-topic"); assertFalse(topicMetric1.getTags().containsKey("hello")); topicMetric1.addTag("hello", "world"); assertTrue(topicMetric1.getTags().containsKey("hello")); assertTrue(dataFeedMetrics.getMetricSet().contains(topicMetric1)); }
@Test public void testCreateTopicLongMetric() { Metric topicMetric1 = dataFeedMetrics.createLongMetric(DataFeedMetricNames.OUTPUT_ROWCOUNT, 100l, Maps.newHashMap()); Metric topicMetric2 = dataFeedMetrics.createLongMetric(DataFeedMetricNames.SIZE_BYTES, 1028l, Maps.newHashMap()); assertEquals("output_rowcount", topicMetric1.getMetricName()); assertEquals("size_bytes", topicMetric2.getMetricName()); assertEquals(new Long(100), topicMetric1.getMetricValue()); assertEquals(new Long(1028), topicMetric2.getMetricValue()); assertTrue(topicMetric1.getTags().containsKey("job")); assertEquals(topicMetric1.getTags().get("job"), "test-job"); assertTrue(topicMetric1.getTags().containsKey("datafeed")); assertEquals(topicMetric1.getTags().get("datafeed"), "test-topic"); assertFalse(topicMetric1.getTags().containsKey("hello")); topicMetric1.addTag("hello", "world"); assertTrue(topicMetric1.getTags().containsKey("hello")); assertTrue(dataFeedMetrics.getMetricSet().contains(topicMetric1)); assertTrue(dataFeedMetrics.getMetricSet().contains(topicMetric2)); }
final MockHoodieSink hoodieSink = new MockHoodieSink(hoodieConf, hoodieKeyGenerator, jsc.get(), INSERT); final Map<String, String> emptyTags = new HashMap<>(); final DataFeedMetrics dfm = new DataFeedMetrics(JOB_NAME, emptyTags); hoodieSink.setDataFeedMetrics(dfm); final Integer successRecordCount = 10; hoodieSink.write(inputRDD); final Set<Metric> ms = dfm.getMetricSet(); final Map<String, Long> expected = ArrayUtils.<String, Long>toMap( new Object[][] {
public TimerMetric createTimerMetric(@NonNull final String metricName) { final TimerMetric metric = new TimerMetric(metricName); metric.addTags(getBaseTags()); metricSet.add(metric); return metric; }
public LongMetric createLongMetric(@NonNull final String metricName, final long metricValue, @NonNull final Map<String, String> additionalTags) { final LongMetric metric = new LongMetric(metricName, metricValue); metric.addTags(getBaseTags()); metric.addTags(additionalTags); this.metricSet.add(metric); return metric; }
private void logWriteMetrics(final Optional<JavaRDD<WriteStatus>> writesStatuses) { if (writesStatuses.isPresent() && this.dataFeedMetrics.isPresent()) { final LongAccumulator totalCount = writesStatuses.get().rdd().sparkContext().longAccumulator(); final LongAccumulator errorCount = writesStatuses.get().rdd().sparkContext().longAccumulator(); writesStatuses.get().foreach(writeStatus -> { errorCount.add(writeStatus.getFailedRecords().size()); totalCount.add(writeStatus.getTotalRecords()); }); this.dataFeedMetrics.get().createLongMetric(DataFeedMetricNames.ERROR_ROWCOUNT, errorCount.value(), this.dataFeedMetricsTags); this.dataFeedMetrics.get().createLongMetric(DataFeedMetricNames.OUTPUT_ROWCOUNT, totalCount.value() - errorCount.value(), this.dataFeedMetricsTags); } }
public TestParentSubDag(@NonNull final AtomicInteger stepCounter) { super("parent"); this.stepCounter = stepCounter; addSubDag(0, new TestSubDag(true, stepCounter)); addSubDag(1, new TestSubDag(false, stepCounter)); final String jobName = "testJob"; setJobMetrics(new JobMetrics(jobName)); setDataFeedMetrics(new DataFeedMetrics(jobName, new HashMap<>())); }
public TimerMetric createTimerMetric(@NonNull final String metricName, @NonNull final Map<String, String> additionalTags, @NonNull final Optional<Instant> startTime) { final TimerMetric metric = startTime.isPresent() ? new TimerMetric(metricName, additionalTags, startTime.get()) : new TimerMetric(metricName); metric.addTags(getBaseTags()); metric.addTags(additionalTags); metricSet.add(metric); return metric; }
tags.put(TABLE_NAME_TAG, this.conf.getKeyspace() + StringTypes.UNDERSCORE + this.conf.getTableName()); this.tableMetrics.get() .createLongMetric(DataFeedMetricNames.OUTPUT_ROWCOUNT, payloadWrapper.getCount(), tags); .createLongMetric(DataFeedMetricNames.OUTPUT_BYTE_SIZE, cassandraPayloadRddSizeEstimator.estimateTotalSize(payloadWrapper), tags);
private void reportStatus(final boolean successful) { final long statusValue = successful ? DataFeedMetricNames.RESULT_SUCCESS : DataFeedMetricNames.RESULT_FAILURE; final LongMetric successMetric = new LongMetric( DataFeedMetricNames.RESULT, statusValue); successMetric.addTags(this.dataFeedMetrics.getBaseTags()); this.reporters.report(successMetric); } }
final RDDWrapper<String> dataWrapper = new RDDWrapper<>(dataToWrite); final long totalRows = dataWrapper.getCount(); this.dataFeedMetrics.get().createLongMetric(DataFeedMetricNames.OUTPUT_ROWCOUNT, totalRows, tags);
tags.put(TABLE_NAME_TAG, this.conf.getKeyspace() + StringTypes.UNDERSCORE + this.conf.getTableName()); this.tableMetrics.get() .createLongMetric(DataFeedMetricNames.OUTPUT_ROWCOUNT, payloadWrapper.getCount(), tags);