return builder.addTags(context.getTags()).addTags(newTags).build();
public JobCatalogBase(Optional<Logger> log, Optional<MetricContext> parentMetricContext, boolean instrumentationEnabled, Optional<Config> sysConfig) { this.log = log.isPresent() ? log.get() : LoggerFactory.getLogger(getClass()); this.listeners = new JobCatalogListenersList(log); if (instrumentationEnabled) { MetricContext realParentCtx = parentMetricContext.or(Instrumented.getMetricContext(new org.apache.gobblin.configuration.State(), getClass())); this.metricContext = realParentCtx.childBuilder(JobCatalog.class.getSimpleName()).build(); this.metrics = createStandardMetrics(sysConfig); this.addListener(this.metrics); } else { this.metricContext = null; this.metrics = null; } }
public MetricContext getDefaultMetricContext() { if (_gobblinEnv.isPresent()) { return _gobblinEnv.get().getMetricContext() .childBuilder(JobExecutionLauncher.class.getSimpleName()).build(); } org.apache.gobblin.configuration.State fakeState = new org.apache.gobblin.configuration.State(getSysConfig().getConfigAsProperties()); List<Tag<?>> tags = new ArrayList<>(); MetricContext res = Instrumented.getMetricContext(fakeState, Launcher.class, tags); return res; }
@Test public void testSimpleEvent() throws IOException { try ( MetricContext metricContext = MetricContext.builder(this.getClass().getCanonicalName() + ".testGraphiteReporter1").build(); GraphiteEventReporter graphiteEventReporter = getBuilder(metricContext).withEmitValueAsKey(false).build();) { Map<String, String> metadata = Maps.newHashMap(); metadata.put(JobEvent.METADATA_JOB_ID, "job1"); metadata.put(TaskEvent.METADATA_TASK_ID, "task1"); metricContext.submitEvent(GobblinTrackingEvent.newBuilder() .setName(JobEvent.TASKS_SUBMITTED) .setNamespace(NAMESPACE) .setMetadata(metadata).build()); try { Thread.sleep(100); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } graphiteEventReporter.report(); try { Thread.sleep(100); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } TimestampedValue retrievedEvent = graphiteSender.getMetric("gobblin.metrics.job1.task1.events.TasksSubmitted"); Assert.assertEquals(retrievedEvent.getValue(), "0"); Assert.assertTrue(retrievedEvent.getTimestamp() <= (System.currentTimeMillis() / 1000l)); } }
@Test public void testReport() throws IOException { MetricContext testContext = MetricContext.builder(getClass().getCanonicalName()).build(); FileSystem fs = mock(FileSystem.class); Path failureLogPath = mock(Path.class);
@Builder private MetricsUpdater(int depth, int counters, int meters, int histograms, int timers, MetricContext baseContext) { this.depth = depth; this.random = new Random(); MetricContext tmpContext = baseContext == null ? RootMetricContext.get() : baseContext; while(depth > 0) { tmpContext = tmpContext.childBuilder(UUID.randomUUID().toString()).build(); depth--; } this.context = tmpContext; this.counters = Lists.newArrayList(); for(int i = 0; i < counters; i++) { this.counters.add(this.context.counter("gobblin.performance.test.counter" + i)); } this.meters = Lists.newArrayList(); for(int i = 0; i < meters; i++) { this.meters.add(this.context.meter("gobblin.performance.test.meter" + i)); } this.histograms = Lists.newArrayList(); for(int i = 0; i < histograms; i++) { this.histograms.add(this.context.histogram("gobblin.performance.test.histogram" + i)); } this.timers = Lists.newArrayList(); for(int i = 0; i < timers; i++) { this.timers.add(this.context.timer("gobblin.performance.test.timer" + i)); } }
@BeforeClass @SuppressWarnings("unchecked") public void setUp() { TaskInputOutputContext<Object, Object, Object, Object> mockContext = Mockito.mock(TaskInputOutputContext.class); this.recordsProcessedCount = Mockito.mock(Counter.class); Mockito.when(mockContext.getCounter( this.name, MetricRegistry.name(RECORDS_PROCESSED, Measurements.COUNT.getName()))) .thenReturn(this.recordsProcessedCount); this.recordProcessRateCount = Mockito.mock(Counter.class); Mockito.when(mockContext.getCounter( this.name, MetricRegistry.name(RECORD_PROCESS_RATE, Measurements.COUNT.getName()))) .thenReturn(this.recordProcessRateCount); this.recordSizeDistributionCount = Mockito.mock(Counter.class); Mockito.when(mockContext.getCounter( this.name, MetricRegistry.name(RECORD_SIZE_DISTRIBUTION, Measurements.COUNT.getName()))) .thenReturn(this.recordSizeDistributionCount); this.totalDurationCount = Mockito.mock(Counter.class); Mockito.when(mockContext.getCounter( this.name, MetricRegistry.name(TOTAL_DURATION, Measurements.COUNT.getName()))) .thenReturn(this.totalDurationCount); this.queueSize = Mockito.mock(Counter.class); Mockito.when(mockContext.getCounter(this.name, QUEUE_SIZE)).thenReturn(this.queueSize); this.hadoopCounterReporter = NewAPIHadoopCounterReporter.builder(mockContext) .convertRatesTo(TimeUnit.SECONDS) .convertDurationsTo(TimeUnit.SECONDS) .filter(MetricFilter.ALL) .build(MetricContext.builder(this.name).build()); }
public MetricsFileSystemInstrumentation(FileSystem underlying) { super(underlying); this.closer = Closer.create(); this.metricContext = new MetricContext.Builder(underlying.getUri() + "_metrics").build(); this.metricContext = this.closer.register(metricContext); this.listStatusTimer = this.metricContext.timer("listStatus"); this.listFilesTimer = this.metricContext.timer("listFiles"); this.globStatusTimer = this.metricContext.timer("globStatus"); this.mkdirTimer = this.metricContext.timer("mkdirs"); this.renameTimer = this.metricContext.timer("rename"); this.deleteTimer = this.metricContext.timer("delete"); this.createTimer = this.metricContext.timer("create"); this.openTimer = this.metricContext.timer("open"); this.setOwnerTimer = this.metricContext.timer("setOwner"); this.getFileStatusTimer = this.metricContext.timer("getFileStatus"); this.setPermissionTimer = this.metricContext.timer("setPermission"); this.setTimesTimer = this.metricContext.timer("setTimes"); this.appendTimer = this.metricContext.timer ("append"); this.concatTimer = this.metricContext.timer ("concat"); this.allTimers = ImmutableList.<ContextAwareTimer>builder().add( this.listStatusTimer, this.listFilesTimer, this.globStatusTimer, this.mkdirTimer, this.renameTimer, this.deleteTimer, this.createTimer, this.openTimer, this.setOwnerTimer, this.getFileStatusTimer, this.setPermissionTimer, this.setTimesTimer, this.appendTimer, this.concatTimer ).build(); }
@Test public void kafkaReporterTagsTest() throws IOException { MetricContext metricContext = MetricContext.builder(this.getClass().getCanonicalName() + ".kafkaReporterTagsTest").build(); Counter counter = metricContext.counter("com.linkedin.example.counter"); Tag<?> tag1 = new Tag<>("tag1", "value1"); Tag<?> tag2 = new Tag<>("tag2", 2); MockKafkaPusher pusher = new MockKafkaPusher(); KafkaReporter kafkaReporter = getBuilder(pusher).withTags(Lists.newArrayList(tag1, tag2)).build("localhost:0000", "topic", new Properties()); counter.inc(); kafkaReporter.report(metricContext); try { Thread.sleep(1000); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } MetricReport metricReport = nextReport(pusher.messageIterator()); Assert.assertEquals(4, metricReport.getTags().size()); Assert.assertTrue(metricReport.getTags().containsKey(tag1.getKey())); Assert.assertEquals(metricReport.getTags().get(tag1.getKey()), tag1.getValue().toString()); Assert.assertTrue(metricReport.getTags().containsKey(tag2.getKey())); Assert.assertEquals(metricReport.getTags().get(tag2.getKey()), tag2.getValue().toString()); }
@Test public void testWithTags() throws IOException { try ( MetricContext metricContext = MetricContext.builder(this.getClass().getCanonicalName() + ".testGraphiteReporter") .addTag(new Tag<String>("taskId", "task_testjob_123")) .addTag(new Tag<String>("forkBranchName", "fork_1")).build(); InfluxDBReporter influxDBReporter = InfluxDBReporter.Factory.newBuilder() .withInfluxDBPusher(influxDBPusher) .withMetricContextName(CONTEXT_NAME) .build(new Properties());) { Counter counter = metricContext.counter(MetricRegistry.name(METRIC_PREFIX, COUNTER)); counter.inc(5l); influxDBReporter.report(new TreeMap<String, Gauge>(), metricContext.getCounters(), new TreeMap<String, Histogram>(), new TreeMap<String, Meter>(), new TreeMap<String, Timer>(), metricContext.getTagMap()); //InfluxDB converts all values to float64 internally Assert.assertEquals(getMetricValue("task_testjob_123.fork_1." + METRIC_PREFIX, COUNTER, Measurements.COUNT), Float.toString(5f)); } }
@Test public void testTags() throws IOException { MetricContext metricContext = MetricContext.builder(this.getClass().getCanonicalName()).build(); Counter counter = metricContext.counter("com.linkedin.example.counter"); Map<String, String> tags = new HashMap<>(); tags.put("testKey", "testValue"); tags.put("key2", "value2"); OutputStreamReporter reporter = OutputStreamReporter.Factory.newBuilder().withTags(tags).outputTo(this.stream).build(new Properties()); counter.inc(); reporter.report(); Assert.assertTrue(this.stream.toString().contains("key2=value2")); Assert.assertTrue(this.stream.toString().contains("testKey=testValue")); String[] lines = this.stream.toString().split("\n"); Map<String, Set<String>> expected = new HashMap<>(); expectMetrics(expected, lines); Set<String> counterSubMetrics = new HashSet<>(); counterSubMetrics.add("count"); expected.put("com.linkedin.example.counter", counterSubMetrics); reporter.close(); }
@Test public void testWithTags() throws IOException { try ( MetricContext metricContext = MetricContext.builder(this.getClass().getCanonicalName() + ".testGraphiteReporter") .addTag(new Tag<String>("taskId", "task_testjob_123")) .addTag(new Tag<String>("forkBranchName", "fork_1")).build(); GraphiteReporter graphiteReporter = GraphiteReporter.Factory.newBuilder() .withGraphitePusher(graphitePusher) .withMetricContextName(CONTEXT_NAME) .build(new Properties());) { Counter counter = metricContext.counter(MetricRegistry.name(METRIC_PREFIX, COUNTER)); counter.inc(5l); graphiteReporter.report(new TreeMap<String, Gauge>(), metricContext.getCounters(), new TreeMap<String, Histogram>(), new TreeMap<String, Meter>(), new TreeMap<String, Timer>(), metricContext.getTagMap()); Assert.assertEquals(getMetricValue("task_testjob_123.fork_1." + METRIC_PREFIX, COUNTER, Measurements.COUNT), Long.toString(5l)); } }
@Test public void kafkaReporterContextTest() throws IOException { Tag<?> tag1 = new Tag<>("tag1", "value1"); MetricContext context = MetricContext.builder("context").addTag(tag1).build(); Counter counter = context.counter("com.linkedin.example.counter"); MockKafkaPusher pusher = new MockKafkaPusher(); KafkaReporter kafkaReporter = getBuilderFromContext(pusher).build("localhost:0000", "topic", new Properties()); counter.inc(); kafkaReporter.report(context); try { Thread.sleep(1000); } catch (InterruptedException ex) { Thread.currentThread().interrupt(); } MetricReport metricReport = nextReport(pusher.messageIterator()); Assert.assertEquals(3, metricReport.getTags().size()); Assert.assertTrue(metricReport.getTags().containsKey(tag1.getKey())); Assert.assertEquals(metricReport.getTags().get(tag1.getKey()), tag1.getValue().toString()); }
@Test public void testTagsFromContext() throws IOException { Tag<?> tag1 = new Tag<>("tag1", "value1"); MetricContext context = MetricContext.builder("context").addTag(tag1).build(); Counter counter = context.counter("com.linkedin.example.counter"); OutputStreamReporter reporter = OutputStreamReporter.Factory.newBuilder().outputTo(this.stream).build(new Properties()); counter.inc(); reporter.report(); Assert.assertTrue(this.stream.toString().contains("tag1=value1")); String[] lines = this.stream.toString().split("\n"); Map<String, Set<String>> expected = new HashMap<>(); expectMetrics(expected, lines); Set<String> counterSubMetrics = new HashSet<>(); counterSubMetrics.add("count"); expected.put("com.linkedin.example.counter", counterSubMetrics); reporter.close(); }
@Test public void testChildContext() { this.childContext = this.context.childBuilder(CHILD_CONTEXT_NAME) .addTag(new Tag<String>(TASK_ID_KEY, TASK_ID_PREFIX + 0)) .build(); Assert.assertEquals(this.childContext.getName(), CHILD_CONTEXT_NAME); Assert.assertTrue(this.childContext.getParent().isPresent()); Assert.assertEquals(this.childContext.getParent().get(), this.context); Assert.assertEquals(this.childContext.getTags().size(), 4); Assert.assertEquals(this.childContext.getTags().get(0).getKey(), JOB_ID_KEY); Assert.assertEquals(this.childContext.getTags().get(0).getValue(), JOB_ID_PREFIX + 0); Assert.assertEquals(this.childContext.getTags().get(1).getKey(), MetricContext.METRIC_CONTEXT_ID_TAG_NAME); Assert.assertEquals(this.childContext.getTags().get(2).getKey(), MetricContext.METRIC_CONTEXT_NAME_TAG_NAME); Assert.assertEquals(this.childContext.getTags().get(3).getKey(), TASK_ID_KEY); Assert.assertEquals(this.childContext.getTags().get(3).getValue(), TASK_ID_PREFIX + 0); }
@BeforeClass public void setUp() { String contextName = CONTEXT_NAME + "_" + UUID.randomUUID().toString(); this.context = MetricContext.builder(contextName) .addTag(new Tag<String>(JOB_ID_KEY, JOB_ID_PREFIX + 0)) .build(); Assert.assertEquals(this.context.getName(), contextName); Assert.assertTrue(this.context.getParent().isPresent()); Assert.assertEquals(this.context.getParent().get(), RootMetricContext.get()); Assert.assertEquals(this.context.getTags().size(), 3); // uuid and name tag gets added automatically Assert.assertEquals(this.context.getTags().get(0).getKey(), JOB_ID_KEY); Assert.assertEquals(this.context.getTags().get(0).getValue(), JOB_ID_PREFIX + 0); // Second tag should be uuid Assert.assertTrue(this.context.getTags().get(1).getValue().toString() .matches("[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}")); }
@BeforeClass public void setUp() { this.context = MetricContext.builder(CONTEXT_NAME).build(); this.counter = ContextAwareMetricFactory.DEFAULT_CONTEXT_AWARE_COUNTER_FACTORY.newMetric( this.context, RECORDS_PROCESSED); this.meter = ContextAwareMetricFactory.DEFAULT_CONTEXT_AWARE_METER_FACTORY.newMetric( this.context, RECORD_PROCESS_RATE); this.histogram = ContextAwareMetricFactory.DEFAULT_CONTEXT_AWARE_HISTOGRAM_FACTORY.newMetric( this.context, RECORD_SIZE_DISTRIBUTION); this.timer = ContextAwareMetricFactory.DEFAULT_CONTEXT_AWARE_TIMER_FACTORY.newMetric( this.context, TOTAL_DURATION); }
private void addTask(int taskIndex, CountDownLatch countDownLatch) { // Build the context of this task, which is a child of the job's context. // Tags of the job (parent) context will be inherited automatically. MetricContext taskContext = this.context.childBuilder("Task" + taskIndex) .addTag(new Tag<String>(TASK_ID_KEY, TASK_ID_PREFIX + taskIndex)) .build(); Task task = new Task(taskContext, taskIndex, this.totalRecords, countDownLatch); this.executor.execute(task); }
public ReporterExampleBase(ContextAwareScheduledReporter.Builder reporterBuilder, int tasks, long totalRecords) { this.executor = Executors.newFixedThreadPool(10); this.context = MetricContext.builder("Job") .addTag(new Tag<String>(JobEvent.METADATA_JOB_NAME, "ExampleJob")) .addTag(new Tag<String>(JobEvent.METADATA_JOB_ID, JOB_NAME + "_" + System.currentTimeMillis())) .build(); this.reporterBuilder = reporterBuilder; this.tasks = tasks; this.totalRecords = totalRecords; }