private StreamPartitionCountMonitor getPartitionCountMonitor(Config config, SystemAdmins systemAdmins) { StreamMetadataCache streamMetadata = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance()); Set<SystemStream> inputStreamsToMonitor = new TaskConfigJava(config).getAllInputStreams(); if (inputStreamsToMonitor.isEmpty()) { throw new SamzaException("Input streams to a job can not be empty."); } return new StreamPartitionCountMonitor( inputStreamsToMonitor, streamMetadata, metrics, new JobConfig(config).getMonitorPartitionChangeFrequency(), streamsChanged -> { // Fail the jobs with durable state store. Otherwise, application state.status remains UNDEFINED s.t. YARN job will be restarted if (hasDurableStores) { log.error("Input topic partition count changed in a job with durable state. Failing the job."); state.status = SamzaApplicationState.SamzaAppStatus.FAILED; } coordinatorException = new PartitionChangeException("Input topic partition count changes detected."); }); }
private StreamPartitionCountMonitor getPartitionCountMonitor(Config config, SystemAdmins systemAdmins) { StreamMetadataCache streamMetadata = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance()); Set<SystemStream> inputStreamsToMonitor = new TaskConfigJava(config).getAllInputStreams(); if (inputStreamsToMonitor.isEmpty()) { throw new SamzaException("Input streams to a job can not be empty."); } return new StreamPartitionCountMonitor( inputStreamsToMonitor, streamMetadata, metrics, new JobConfig(config).getMonitorPartitionChangeFrequency(), streamsChanged -> { // Fail the jobs with durable state store. Otherwise, application state.status remains UNDEFINED s.t. YARN job will be restarted if (hasDurableStores) { log.error("Input topic partition count changed in a job with durable state. Failing the job."); state.status = SamzaApplicationState.SamzaAppStatus.FAILED; } coordinatorException = new PartitionChangeException("Input topic partition count changes detected."); }); }
private StreamPartitionCountMonitor getPartitionCountMonitor(Config config, SystemAdmins systemAdmins) { StreamMetadataCache streamMetadata = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance()); Set<SystemStream> inputStreamsToMonitor = new TaskConfigJava(config).getAllInputStreams(); if (inputStreamsToMonitor.isEmpty()) { throw new SamzaException("Input streams to a job can not be empty."); } return new StreamPartitionCountMonitor( inputStreamsToMonitor, streamMetadata, metrics, new JobConfig(config).getMonitorPartitionChangeFrequency(), streamsChanged -> { // Fail the jobs with durable state store. Otherwise, application state.status remains UNDEFINED s.t. YARN job will be restarted if (hasDurableStores) { log.error("Input topic partition count changed in a job with durable state. Failing the job."); state.status = SamzaApplicationState.SamzaAppStatus.FAILED; } coordinatorException = new PartitionChangeException("Input topic partition count changes detected."); }); }
private StreamPartitionCountMonitor getPartitionCountMonitor(Config config, SystemAdmins systemAdmins) { StreamMetadataCache streamMetadata = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance()); Set<SystemStream> inputStreamsToMonitor = new TaskConfigJava(config).getAllInputStreams(); if (inputStreamsToMonitor.isEmpty()) { throw new SamzaException("Input streams to a job can not be empty."); } return new StreamPartitionCountMonitor( inputStreamsToMonitor, streamMetadata, metrics, new JobConfig(config).getMonitorPartitionChangeFrequency(), streamsChanged -> { // Fail the jobs with durable state store. Otherwise, application state.status remains UNDEFINED s.t. YARN job will be restarted if (hasDurableStores) { log.error("Input topic partition count changed in a job with durable state. Failing the job."); state.status = SamzaApplicationState.SamzaAppStatus.FAILED; } coordinatorException = new PartitionChangeException("Input topic partition count changes detected."); }); }