public StreamingKafkaSpecConsumer(Config config, MutableJobCatalog jobCatalog, Optional<Logger> log) { String topic = config.getString(SPEC_KAFKA_TOPICS_KEY); Config defaults = ConfigFactory.parseMap(ImmutableMap.of(AvroJobSpecKafkaJobMonitor.TOPIC_KEY, topic, KafkaJobMonitor.KAFKA_AUTO_OFFSET_RESET_KEY, KafkaJobMonitor.KAFKA_AUTO_OFFSET_RESET_SMALLEST)); try { _jobMonitor = (AvroJobSpecKafkaJobMonitor)(new AvroJobSpecKafkaJobMonitor.Factory()) .forConfig(config.withFallback(defaults), jobCatalog); } catch (IOException e) { throw new RuntimeException("Could not create job monitor", e); } _jobCatalog = jobCatalog; _jobSpecQueue = new LinkedBlockingQueue<>(ConfigUtils.getInt(config, "SPEC_STREAMING_BLOCKING_QUEUE_SIZE", DEFAULT_SPEC_STREAMING_BLOCKING_QUEUE_SIZE)); _metricContext = Instrumented.getMetricContext(ConfigUtils.configToState(config), this.getClass()); _metrics = new Metrics(this._metricContext); }
public StreamingKafkaSpecConsumer(Config config, MutableJobCatalog jobCatalog, Optional<Logger> log) { String topic = config.getString(SPEC_KAFKA_TOPICS_KEY); Config defaults = ConfigFactory.parseMap(ImmutableMap.of(AvroJobSpecKafkaJobMonitor.TOPIC_KEY, topic, KafkaJobMonitor.KAFKA_AUTO_OFFSET_RESET_KEY, KafkaJobMonitor.KAFKA_AUTO_OFFSET_RESET_SMALLEST)); try { _jobMonitor = (AvroJobSpecKafkaJobMonitor)(new AvroJobSpecKafkaJobMonitor.Factory()) .forConfig(config.withFallback(defaults), jobCatalog); } catch (IOException e) { throw new RuntimeException("Could not create job monitor", e); } _jobCatalog = jobCatalog; _jobSpecQueue = new LinkedBlockingQueue<>(ConfigUtils.getInt(config, "SPEC_STREAMING_BLOCKING_QUEUE_SIZE", DEFAULT_SPEC_STREAMING_BLOCKING_QUEUE_SIZE)); _metricContext = Instrumented.getMetricContext(ConfigUtils.configToState(config), this.getClass()); _metrics = new Metrics(this._metricContext); }
@Override public JobSpecMonitor forJobCatalog(GobblinInstanceDriver instanceDriver, MutableJobCatalog jobCatalog) throws IOException { Config config = instanceDriver.getSysConfig().getConfig().getConfig(CONFIG_PREFIX).withFallback(DEFAULTS); return forConfig(config, jobCatalog); }
@Override public JobSpecMonitor forJobCatalog(GobblinInstanceDriver instanceDriver, MutableJobCatalog jobCatalog) throws IOException { Config config = instanceDriver.getSysConfig().getConfig().getConfig(CONFIG_PREFIX).withFallback(DEFAULTS); return forConfig(config, jobCatalog); }