/** * Create a {@link AvroJobSpecKafkaJobMonitor} from an input {@link Config}. Useful for multiple monitors, where * the configuration of each monitor is scoped. * @param localScopeConfig The sub-{@link Config} for this monitor without any namespacing (e.g. the key for * topic should simply be "topic"). * @throws IOException */ public JobSpecMonitor forConfig(Config localScopeConfig, MutableJobCatalog jobCatalog) throws IOException { Preconditions.checkArgument(localScopeConfig.hasPath(TOPIC_KEY)); Config config = localScopeConfig.withFallback(DEFAULTS); String topic = config.getString(TOPIC_KEY); SchemaVersionWriter versionWriter; try { versionWriter = (SchemaVersionWriter) GobblinConstructorUtils. invokeLongestConstructor(Class.forName(config.getString(SCHEMA_VERSION_READER_CLASS)), config); } catch (ReflectiveOperationException roe) { throw new IllegalArgumentException(roe); } return new AvroJobSpecKafkaJobMonitor(topic, jobCatalog, config, versionWriter); } }
private long getMessageParseFailures() { return StreamingKafkaSpecConsumer.this._jobMonitor.getMessageParseFailures() != null? StreamingKafkaSpecConsumer.this._jobMonitor.getMessageParseFailures().getCount():0; } }
private long getNewSpecs() { return StreamingKafkaSpecConsumer.this._jobMonitor.getNewSpecs() != null? StreamingKafkaSpecConsumer.this._jobMonitor.getNewSpecs().getCount() : 0; }
private long getRemovedSpecs() { return StreamingKafkaSpecConsumer.this._jobMonitor.getRemovedSpecs() != null? StreamingKafkaSpecConsumer.this._jobMonitor.getRemovedSpecs().getCount() : 0; }
@Override protected void startUp() { // listener will add job specs to a blocking queue to send to callers of changedSpecs() // IMPORTANT: This addListener should be invoked after job catalog has been initialized. This is guaranteed because // StreamingKafkaSpecConsumer is boot after jobCatalog in GobblinClusterManager::startAppLauncherAndServices() _jobCatalog.addListener(new JobSpecListener()); _jobMonitor.startAsync().awaitRunning(); }
@Override protected void shutDown() { _jobMonitor.stopAsync().awaitTerminated(); }
@Override protected void processMessage(MessageAndMetadata<byte[], byte[]> message) { try { Collection<Either<JobSpec, URI>> parsedCollection = parseJobSpec(message.message()); for (Either<JobSpec, URI> parsedMessage : parsedCollection) { JobSpec jobSpec = ((Either.Left<JobSpec, URI>)parsedMessage).getLeft();
private long getRemovedSpecs() { return StreamingKafkaSpecConsumer.this._jobMonitor.getRemovedSpecs() != null? StreamingKafkaSpecConsumer.this._jobMonitor.getRemovedSpecs().getCount() : 0; }
@Override protected void startUp() { // listener will add job specs to a blocking queue to send to callers of changedSpecs() // IMPORTANT: This addListener should be invoked after job catalog has been initialized. This is guaranteed because // StreamingKafkaSpecConsumer is boot after jobCatalog in GobblinClusterManager::startAppLauncherAndServices() _jobCatalog.addListener(new JobSpecListener()); _jobMonitor.startAsync().awaitRunning(); }
@Override protected void shutDown() { _jobMonitor.stopAsync().awaitTerminated(); }
@Override protected void processMessage(MessageAndMetadata<byte[], byte[]> message) { try { Collection<Either<JobSpec, URI>> parsedCollection = parseJobSpec(message.message()); for (Either<JobSpec, URI> parsedMessage : parsedCollection) { JobSpec jobSpec = ((Either.Left<JobSpec, URI>)parsedMessage).getLeft();
private long getMessageParseFailures() { return StreamingKafkaSpecConsumer.this._jobMonitor.getMessageParseFailures() != null? StreamingKafkaSpecConsumer.this._jobMonitor.getMessageParseFailures().getCount():0; } }
private long getNewSpecs() { return StreamingKafkaSpecConsumer.this._jobMonitor.getNewSpecs() != null? StreamingKafkaSpecConsumer.this._jobMonitor.getNewSpecs().getCount() : 0; }
/** * Create a {@link AvroJobSpecKafkaJobMonitor} from an input {@link Config}. Useful for multiple monitors, where * the configuration of each monitor is scoped. * @param localScopeConfig The sub-{@link Config} for this monitor without any namespacing (e.g. the key for * topic should simply be "topic"). * @throws IOException */ public JobSpecMonitor forConfig(Config localScopeConfig, MutableJobCatalog jobCatalog) throws IOException { Preconditions.checkArgument(localScopeConfig.hasPath(TOPIC_KEY)); Config config = localScopeConfig.withFallback(DEFAULTS); String topic = config.getString(TOPIC_KEY); SchemaVersionWriter versionWriter; try { versionWriter = (SchemaVersionWriter) GobblinConstructorUtils. invokeLongestConstructor(Class.forName(config.getString(SCHEMA_VERSION_READER_CLASS)), config); } catch (ReflectiveOperationException roe) { throw new IllegalArgumentException(roe); } return new AvroJobSpecKafkaJobMonitor(topic, jobCatalog, config, versionWriter); } }