/** * @return the set of SystemStreams which were configured as bootstrap streams. */ public Set<SystemStream> getBootstrapStreams() { Set<SystemStream> bootstrapInputs = new HashSet<>(); Set<SystemStream> allInputs = taskConfigJava.getAllInputStreams(); for (SystemStream systemStream : allInputs) { if (streamConfig.getBootstrapEnabled(systemStream)) { bootstrapInputs.add(systemStream); } } return Collections.unmodifiableSet(bootstrapInputs); }
/** * @return the set of SystemStreams which were configured as bootstrap streams. */ public Set<SystemStream> getBootstrapStreams() { Set<SystemStream> bootstrapInputs = new HashSet<>(); Set<SystemStream> allInputs = taskConfigJava.getAllInputStreams(); for (SystemStream systemStream : allInputs) { if (streamConfig.getBootstrapEnabled(systemStream)) { bootstrapInputs.add(systemStream); } } return Collections.unmodifiableSet(bootstrapInputs); }
/** * @return the set of SystemStreams which were configured as bootstrap streams. */ public Set<SystemStream> getBootstrapStreams() { Set<SystemStream> bootstrapInputs = new HashSet<>(); Set<SystemStream> allInputs = taskConfigJava.getAllInputStreams(); for (SystemStream systemStream : allInputs) { if (streamConfig.getBootstrapEnabled(systemStream)) { bootstrapInputs.add(systemStream); } } return Collections.unmodifiableSet(bootstrapInputs); }
/** * @return the set of SystemStreams which were configured as bootstrap streams. */ public Set<SystemStream> getBootstrapStreams() { Set<SystemStream> bootstrapInputs = new HashSet<>(); Set<SystemStream> allInputs = taskConfigJava.getAllInputStreams(); for (SystemStream systemStream : allInputs) { if (streamConfig.getBootstrapEnabled(systemStream)) { bootstrapInputs.add(systemStream); } } return Collections.unmodifiableSet(bootstrapInputs); }
/** * @return the set of SystemStreams which were configured as bootstrap streams. */ public Set<SystemStream> getBootstrapStreams() { Set<SystemStream> bootstrapInputs = new HashSet<>(); Set<SystemStream> allInputs = taskConfigJava.getAllInputStreams(); for (SystemStream systemStream : allInputs) { if (streamConfig.getBootstrapEnabled(systemStream)) { bootstrapInputs.add(systemStream); } } return Collections.unmodifiableSet(bootstrapInputs); }
/** * Gets the priority of every SystemStream for which the priority * was explicitly configured with a value >=0. * * @return the explicitly-configured stream priorities as a map from * SystemStream to the configured priority value. Streams that * were not explicitly configured with a priority are not returned. */ public Map<SystemStream, Integer> getPriorityStreams() { Set<SystemStream> allInputs = taskConfigJava.getAllInputStreams(); Map<SystemStream, Integer> priorityStreams = new HashMap<>(); for (SystemStream systemStream : allInputs) { int priority = streamConfig.getPriority(systemStream); if (priority >= 0) { priorityStreams.put(systemStream, priority); } } return Collections.unmodifiableMap(priorityStreams); } }
/** * Gets the priority of every SystemStream for which the priority * was explicitly configured with a value >=0. * * @return the explicitly-configured stream priorities as a map from * SystemStream to the configured priority value. Streams that * were not explicitly configured with a priority are not returned. */ public Map<SystemStream, Integer> getPriorityStreams() { Set<SystemStream> allInputs = taskConfigJava.getAllInputStreams(); Map<SystemStream, Integer> priorityStreams = new HashMap<>(); for (SystemStream systemStream : allInputs) { int priority = streamConfig.getPriority(systemStream); if (priority >= 0) { priorityStreams.put(systemStream, priority); } } return Collections.unmodifiableMap(priorityStreams); } }
/** * Gets the priority of every SystemStream for which the priority * was explicitly configured with a value >=0. * * @return the explicitly-configured stream priorities as a map from * SystemStream to the configured priority value. Streams that * were not explicitly configured with a priority are not returned. */ public Map<SystemStream, Integer> getPriorityStreams() { Set<SystemStream> allInputs = taskConfigJava.getAllInputStreams(); Map<SystemStream, Integer> priorityStreams = new HashMap<>(); for (SystemStream systemStream : allInputs) { int priority = streamConfig.getPriority(systemStream); if (priority >= 0) { priorityStreams.put(systemStream, priority); } } return Collections.unmodifiableMap(priorityStreams); } }
/** * Gets the priority of every SystemStream for which the priority * was explicitly configured with a value >=0. * * @return the explicitly-configured stream priorities as a map from * SystemStream to the configured priority value. Streams that * were not explicitly configured with a priority are not returned. */ public Map<SystemStream, Integer> getPriorityStreams() { Set<SystemStream> allInputs = taskConfigJava.getAllInputStreams(); Map<SystemStream, Integer> priorityStreams = new HashMap<>(); for (SystemStream systemStream : allInputs) { int priority = streamConfig.getPriority(systemStream); if (priority >= 0) { priorityStreams.put(systemStream, priority); } } return Collections.unmodifiableMap(priorityStreams); } }
/** * Gets the priority of every SystemStream for which the priority * was explicitly configured with a value >=0. * * @return the explicitly-configured stream priorities as a map from * SystemStream to the configured priority value. Streams that * were not explicitly configured with a priority are not returned. */ public Map<SystemStream, Integer> getPriorityStreams() { Set<SystemStream> allInputs = taskConfigJava.getAllInputStreams(); Map<SystemStream, Integer> priorityStreams = new HashMap<>(); for (SystemStream systemStream : allInputs) { int priority = streamConfig.getPriority(systemStream); if (priority >= 0) { priorityStreams.put(systemStream, priority); } } return Collections.unmodifiableMap(priorityStreams); } }
private Optional<StreamPartitionCountMonitor> getPartitionCountMonitor(Config config, SystemAdmins systemAdmins) { StreamMetadataCache streamMetadata = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance()); Set<SystemStream> inputStreamsToMonitor = new TaskConfigJava(config).getAllInputStreams(); if (inputStreamsToMonitor.isEmpty()) { throw new SamzaException("Input streams to a job can not be empty."); } return Optional.of(new StreamPartitionCountMonitor(inputStreamsToMonitor, streamMetadata, metrics, new JobConfig(config).getMonitorPartitionChangeFrequency(), streamsChanged -> { // Fail the jobs with durable state store. Otherwise, application state.status remains UNDEFINED s.t. YARN job will be restarted if (hasDurableStores) { log.error("Input topic partition count changed in a job with durable state. Failing the job."); state.status = SamzaApplicationState.SamzaAppStatus.FAILED; } coordinatorException = new PartitionChangeException("Input topic partition count changes detected."); })); }
private StreamPartitionCountMonitor getPartitionCountMonitor(Config config, SystemAdmins systemAdmins) { StreamMetadataCache streamMetadata = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance()); Set<SystemStream> inputStreamsToMonitor = new TaskConfigJava(config).getAllInputStreams(); if (inputStreamsToMonitor.isEmpty()) { throw new SamzaException("Input streams to a job can not be empty."); } return new StreamPartitionCountMonitor( inputStreamsToMonitor, streamMetadata, metrics, new JobConfig(config).getMonitorPartitionChangeFrequency(), streamsChanged -> { // Fail the jobs with durable state store. Otherwise, application state.status remains UNDEFINED s.t. YARN job will be restarted if (hasDurableStores) { log.error("Input topic partition count changed in a job with durable state. Failing the job."); state.status = SamzaApplicationState.SamzaAppStatus.FAILED; } coordinatorException = new PartitionChangeException("Input topic partition count changes detected."); }); }
Set<SystemStream> inputStreamsToMonitor = new TaskConfigJava(config).getAllInputStreams(); if (inputStreamsToMonitor.isEmpty()) { throw new SamzaException("Input streams to a job can not be empty.");
private StreamPartitionCountMonitor getPartitionCountMonitor(Config config, SystemAdmins systemAdmins) { StreamMetadataCache streamMetadata = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance()); Set<SystemStream> inputStreamsToMonitor = new TaskConfigJava(config).getAllInputStreams(); if (inputStreamsToMonitor.isEmpty()) { throw new SamzaException("Input streams to a job can not be empty."); } return new StreamPartitionCountMonitor( inputStreamsToMonitor, streamMetadata, metrics, new JobConfig(config).getMonitorPartitionChangeFrequency(), streamsChanged -> { // Fail the jobs with durable state store. Otherwise, application state.status remains UNDEFINED s.t. YARN job will be restarted if (hasDurableStores) { log.error("Input topic partition count changed in a job with durable state. Failing the job."); state.status = SamzaApplicationState.SamzaAppStatus.FAILED; } coordinatorException = new PartitionChangeException("Input topic partition count changes detected."); }); }
private StreamPartitionCountMonitor getPartitionCountMonitor(Config config, SystemAdmins systemAdmins) { StreamMetadataCache streamMetadata = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance()); Set<SystemStream> inputStreamsToMonitor = new TaskConfigJava(config).getAllInputStreams(); if (inputStreamsToMonitor.isEmpty()) { throw new SamzaException("Input streams to a job can not be empty."); } return new StreamPartitionCountMonitor( inputStreamsToMonitor, streamMetadata, metrics, new JobConfig(config).getMonitorPartitionChangeFrequency(), streamsChanged -> { // Fail the jobs with durable state store. Otherwise, application state.status remains UNDEFINED s.t. YARN job will be restarted if (hasDurableStores) { log.error("Input topic partition count changed in a job with durable state. Failing the job."); state.status = SamzaApplicationState.SamzaAppStatus.FAILED; } coordinatorException = new PartitionChangeException("Input topic partition count changes detected."); }); }
private StreamPartitionCountMonitor getPartitionCountMonitor(Config config, SystemAdmins systemAdmins) { StreamMetadataCache streamMetadata = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance()); Set<SystemStream> inputStreamsToMonitor = new TaskConfigJava(config).getAllInputStreams(); if (inputStreamsToMonitor.isEmpty()) { throw new SamzaException("Input streams to a job can not be empty."); } return new StreamPartitionCountMonitor( inputStreamsToMonitor, streamMetadata, metrics, new JobConfig(config).getMonitorPartitionChangeFrequency(), streamsChanged -> { // Fail the jobs with durable state store. Otherwise, application state.status remains UNDEFINED s.t. YARN job will be restarted if (hasDurableStores) { log.error("Input topic partition count changed in a job with durable state. Failing the job."); state.status = SamzaApplicationState.SamzaAppStatus.FAILED; } coordinatorException = new PartitionChangeException("Input topic partition count changes detected."); }); }
@VisibleForTesting StreamPartitionCountMonitor getPartitionCountMonitor() { StreamMetadataCache streamMetadata = new StreamMetadataCache(systemAdmins, 0, SystemClock.instance()); Set<SystemStream> inputStreamsToMonitor = new TaskConfigJava(config).getAllInputStreams(); return new StreamPartitionCountMonitor( inputStreamsToMonitor, streamMetadata, metrics.getMetricsRegistry(), new JobConfig(config).getMonitorPartitionChangeFrequency(), streamsChanged -> { if (leaderElector.amILeader()) { debounceTimer.scheduleAfterDebounceTime(ON_PROCESSOR_CHANGE, 0, this::doOnProcessorChange); } }); }