private HighResolutionClock createHighResClock(Config config) { if (new MetricsConfig(config).getMetricsTimerEnabled()) { return System::nanoTime; } else { return () -> 0; } } }
private HighResolutionClock createHighResClock(Config config) { MetricsConfig metricsConfig = new MetricsConfig(config); // The timer metrics calculation here is only enabled for debugging if (metricsConfig.getMetricsTimerEnabled() && metricsConfig.getMetricsTimerDebugEnabled()) { return System::nanoTime; } else { return () -> 0; } } }
public static Map<String, MetricsReporter> getMetricsReporters(MetricsConfig config, String containerName) { Map<String, MetricsReporter> metricsReporters = new HashMap<>(); for (String metricsReporterName : JavaConverters.seqAsJavaListConverter(config.getMetricReporterNames()).asJava()) { String metricsFactoryClassName = config.getMetricsFactoryClass(metricsReporterName).get(); if (metricsFactoryClassName == null) { throw new SamzaException(String.format("Metrics reporter %s missing .class config", metricsReporterName)); } MetricsReporterFactory metricsReporterFactory = Util.getObj(metricsFactoryClassName, MetricsReporterFactory.class); metricsReporters.put(metricsReporterName, metricsReporterFactory.getMetricsReporter(metricsReporterName, containerName, config)); } return metricsReporters; } }
ReadableMetricsRegistry metricsRegistry = new MetricsRegistryMap(); log.info("Creating new SamzaRestService with config: {}", config); MetricsConfig metricsConfig = new MetricsConfig(config); Map<String, MetricsReporter> metricsReporters = MetricsReporterLoader.getMetricsReporters(metricsConfig, Util.getLocalHost().getHostName());
private void initTables(boolean isTimerMetricsDisabled, ReadWriteTable ... tables) { Map<String, String> config = new HashMap<>(); if (isTimerMetricsDisabled) { config.put(MetricsConfig.METRICS_TIMER_ENABLED(), "false"); } Context context = new MockContext(); doReturn(new MapConfig(config)).when(context.getJobContext()).getConfig(); metricsRegistry = mock(MetricsRegistry.class); doReturn(mock(Timer.class)).when(metricsRegistry).newTimer(anyString(), anyString()); doReturn(mock(Counter.class)).when(metricsRegistry).newCounter(anyString(), anyString()); doReturn(mock(Gauge.class)).when(metricsRegistry).newGauge(anyString(), any()); doReturn(metricsRegistry).when(context.getContainerContext()).getContainerMetricsRegistry(); Arrays.asList(tables).forEach(t -> t.init(context)); }
public static Map<String, MetricsReporter> getMetricsReporters(MetricsConfig config, String containerName) { Map<String, MetricsReporter> metricsReporters = new HashMap<>(); for (String metricsReporterName : JavaConverters.seqAsJavaListConverter(config.getMetricReporterNames()).asJava()) { String metricsFactoryClassName = config.getMetricsFactoryClass(metricsReporterName).get(); if (metricsFactoryClassName == null) { throw new SamzaException(String.format("Metrics reporter %s missing .class config", metricsReporterName)); } MetricsReporterFactory metricsReporterFactory = Util.getObj(metricsFactoryClassName, MetricsReporterFactory.class); metricsReporters.put(metricsReporterName, metricsReporterFactory.getMetricsReporter(metricsReporterName, containerName, config)); } return metricsReporters; } }
ZkJobCoordinator(Config config, MetricsRegistry metricsRegistry, ZkUtils zkUtils) { this.config = config; this.metrics = new ZkJobCoordinatorMetrics(metricsRegistry); this.processorId = createProcessorId(config); this.zkUtils = zkUtils; // setup a listener for a session state change // we are mostly interested in "session closed" and "new session created" events zkUtils.getZkClient().subscribeStateChanges(new ZkSessionStateChangedListener()); leaderElector = new ZkLeaderElector(processorId, zkUtils); leaderElector.setLeaderElectorListener(new LeaderElectorListenerImpl()); this.debounceTimeMs = new JobConfig(config).getDebounceTimeMs(); this.reporters = MetricsReporterLoader.getMetricsReporters(new MetricsConfig(config), processorId); debounceTimer = new ScheduleAfterDebounceTime(processorId); debounceTimer.setScheduledTaskCallback(throwable -> { LOG.error("Received exception in debounce timer! Stopping the job coordinator", throwable); stop(); }); this.barrier = new ZkBarrierForVersionUpgrade(zkUtils.getKeyBuilder().getJobModelVersionBarrierPrefix(), zkUtils, new ZkBarrierListenerImpl(), debounceTimer); systemAdmins = new SystemAdmins(config); streamMetadataCache = new StreamMetadataCache(systemAdmins, METADATA_CACHE_TTL_MS, SystemClock.instance()); }
private LocalTable createTable(boolean isTimerDisabled) { Map<String, String> config = new HashMap<>(); if (isTimerDisabled) { config.put(MetricsConfig.METRICS_TIMER_ENABLED(), "false"); } Context context = mock(Context.class); JobContext jobContext = mock(JobContext.class); when(context.getJobContext()).thenReturn(jobContext); when(jobContext.getConfig()).thenReturn(new MapConfig(config)); ContainerContext containerContext = mock(ContainerContext.class); when(context.getContainerContext()).thenReturn(containerContext); when(containerContext.getContainerMetricsRegistry()).thenReturn(metricsRegistry); LocalTable table = new LocalTable("t1", kvStore); table.init(context); return table; } }
private HighResolutionClock createHighResClock(Config config) { if (new MetricsConfig(config).getMetricsTimerEnabled()) { return System::nanoTime; } else { return () -> 0; } } }
public static Map<String, MetricsReporter> getMetricsReporters(MetricsConfig config, String containerName) { Map<String, MetricsReporter> metricsReporters = new HashMap<>(); for (String metricsReporterName : JavaConverters.seqAsJavaListConverter(config.getMetricReporterNames()).asJava()) { String metricsFactoryClassName = config.getMetricsFactoryClass(metricsReporterName).get(); if (metricsFactoryClassName == null) { throw new SamzaException(String.format("Metrics reporter %s missing .class config", metricsReporterName)); } MetricsReporterFactory metricsReporterFactory = Util.getObj(metricsFactoryClassName, MetricsReporterFactory.class); metricsReporters.put(metricsReporterName, metricsReporterFactory.getMetricsReporter(metricsReporterName, containerName, config)); } return metricsReporters; } }
ZkJobCoordinator(Config config, MetricsRegistry metricsRegistry, ZkUtils zkUtils) { this.config = config; this.metrics = new ZkJobCoordinatorMetrics(metricsRegistry); this.processorId = createProcessorId(config); this.zkUtils = zkUtils; // setup a listener for a session state change // we are mostly interested in "session closed" and "new session created" events zkUtils.getZkClient().subscribeStateChanges(new ZkSessionStateChangedListener()); leaderElector = new ZkLeaderElector(processorId, zkUtils); leaderElector.setLeaderElectorListener(new LeaderElectorListenerImpl()); this.debounceTimeMs = new JobConfig(config).getDebounceTimeMs(); this.reporters = MetricsReporterLoader.getMetricsReporters(new MetricsConfig(config), processorId); debounceTimer = new ScheduleAfterDebounceTime(processorId); debounceTimer.setScheduledTaskCallback(throwable -> { LOG.error("Received exception in debounce timer! Stopping the job coordinator", throwable); stop(); }); this.barrier = new ZkBarrierForVersionUpgrade(zkUtils.getKeyBuilder().getJobModelVersionBarrierPrefix(), zkUtils, new ZkBarrierListenerImpl(), debounceTimer); systemAdmins = new SystemAdmins(config); streamMetadataCache = new StreamMetadataCache(systemAdmins, METADATA_CACHE_TTL_MS, SystemClock.instance()); }
private LocalTable createTable(boolean isTimerDisabled) { Map<String, String> config = new HashMap<>(); if (isTimerDisabled) { config.put(MetricsConfig.METRICS_TIMER_ENABLED(), "false"); } Context context = mock(Context.class); JobContext jobContext = mock(JobContext.class); when(context.getJobContext()).thenReturn(jobContext); when(jobContext.getConfig()).thenReturn(new MapConfig(config)); ContainerContext containerContext = mock(ContainerContext.class); when(context.getContainerContext()).thenReturn(containerContext); when(containerContext.getContainerMetricsRegistry()).thenReturn(metricsRegistry); LocalTable table = new LocalTable("t1", kvStore); table.init(context); return table; } }
private HighResolutionClock createHighResClock(Config config) { if (new MetricsConfig(config).getMetricsTimerEnabled()) { return System::nanoTime; } else { return () -> 0; } } }
public static Map<String, MetricsReporter> getMetricsReporters(MetricsConfig config, String containerName) { Map<String, MetricsReporter> metricsReporters = new HashMap<>(); for (String metricsReporterName : JavaConverters.seqAsJavaListConverter(config.getMetricReporterNames()).asJava()) { String metricsFactoryClassName = config.getMetricsFactoryClass(metricsReporterName).get(); if (metricsFactoryClassName == null) { throw new SamzaException(String.format("Metrics reporter %s missing .class config", metricsReporterName)); } MetricsReporterFactory metricsReporterFactory = Util.getObj(metricsFactoryClassName, MetricsReporterFactory.class); metricsReporters.put(metricsReporterName, metricsReporterFactory.getMetricsReporter(metricsReporterName, containerName, config)); } return metricsReporters; } }
ZkJobCoordinator(Config config, MetricsRegistry metricsRegistry, ZkUtils zkUtils) { this.config = config; this.metrics = new ZkJobCoordinatorMetrics(metricsRegistry); this.processorId = createProcessorId(config); this.zkUtils = zkUtils; // setup a listener for a session state change // we are mostly interested in "session closed" and "new session created" events zkUtils.getZkClient().subscribeStateChanges(new ZkSessionStateChangedListener()); leaderElector = new ZkLeaderElector(processorId, zkUtils); leaderElector.setLeaderElectorListener(new LeaderElectorListenerImpl()); this.debounceTimeMs = new JobConfig(config).getDebounceTimeMs(); this.reporters = MetricsReporterLoader.getMetricsReporters(new MetricsConfig(config), processorId); debounceTimer = new ScheduleAfterDebounceTime(processorId); debounceTimer.setScheduledTaskCallback(throwable -> { LOG.error("Received exception in debounce timer! Stopping the job coordinator", throwable); stop(); }); this.barrier = new ZkBarrierForVersionUpgrade(zkUtils.getKeyBuilder().getJobModelVersionBarrierPrefix(), zkUtils, new ZkBarrierListenerImpl(), debounceTimer); systemAdmins = new SystemAdmins(config); streamMetadataCache = new StreamMetadataCache(systemAdmins, METADATA_CACHE_TTL_MS, SystemClock.instance()); }
private HighResolutionClock createHighResClock(Config config) { if (new MetricsConfig(config).getMetricsTimerEnabled()) { return System::nanoTime; } else { return () -> 0; } } }
public static Map<String, MetricsReporter> getMetricsReporters(MetricsConfig config, String containerName) { Map<String, MetricsReporter> metricsReporters = new HashMap<>(); for (String metricsReporterName : JavaConverters.seqAsJavaListConverter(config.getMetricReporterNames()).asJava()) { String metricsFactoryClassName = config.getMetricsFactoryClass(metricsReporterName).get(); if (metricsFactoryClassName == null) { throw new SamzaException(String.format("Metrics reporter %s missing .class config", metricsReporterName)); } MetricsReporterFactory metricsReporterFactory = Util.getObj(metricsFactoryClassName, MetricsReporterFactory.class); metricsReporters.put(metricsReporterName, metricsReporterFactory.getMetricsReporter(metricsReporterName, containerName, config)); } return metricsReporters; } }
ZkJobCoordinator(Config config, MetricsRegistry metricsRegistry, ZkUtils zkUtils) { this.config = config; this.metrics = new ZkJobCoordinatorMetrics(metricsRegistry); this.processorId = createProcessorId(config); this.zkUtils = zkUtils; // setup a listener for a session state change // we are mostly interested in "session closed" and "new session created" events zkUtils.getZkClient().subscribeStateChanges(new ZkSessionStateChangedListener()); leaderElector = new ZkLeaderElector(processorId, zkUtils); leaderElector.setLeaderElectorListener(new LeaderElectorListenerImpl()); this.debounceTimeMs = new JobConfig(config).getDebounceTimeMs(); this.reporters = MetricsReporterLoader.getMetricsReporters(new MetricsConfig(config), processorId); debounceTimer = new ScheduleAfterDebounceTime(processorId); debounceTimer.setScheduledTaskCallback(throwable -> { LOG.error("Received exception in debounce timer! Stopping the job coordinator", throwable); stop(); }); this.barrier = new ZkBarrierForVersionUpgrade(zkUtils.getKeyBuilder().getJobModelVersionBarrierPrefix(), zkUtils, new ZkBarrierListenerImpl(), debounceTimer); systemAdmins = new SystemAdmins(config); streamMetadataCache = new StreamMetadataCache(systemAdmins, METADATA_CACHE_TTL_MS, SystemClock.instance()); }
@Override public void init(Context context) { MetricsConfig metricsConfig = new MetricsConfig(context.getJobContext().getConfig()); clock = metricsConfig.getMetricsTimerEnabled() ? () -> System.nanoTime() : () -> 0L; metrics = new TableMetrics(context, this, tableId); }
@Override public void init(Context context) { table.init(context); MetricsConfig metricsConfig = new MetricsConfig(context.getJobContext().getConfig()); if (metricsConfig.getMetricsTimerEnabled()) { TableMetricsUtil tableMetricsUtil = new TableMetricsUtil(context, this, tableId); if (isReadRateLimited()) { readRateLimiter.setTimerMetric(tableMetricsUtil.newTimer("get-throttle-ns")); } if (isWriteRateLimited()) { writeRateLimiter.setTimerMetric(tableMetricsUtil.newTimer("put-throttle-ns")); } } }