/** * Get the SystemStreams for the configured input and broadcast streams. * * @return the set of SystemStreams for both standard inputs and broadcast stream inputs. */ public Set<SystemStream> getAllInputStreams() { Set<SystemStream> allInputSS = new HashSet<>(); TaskConfig taskConfig = TaskConfig.Config2Task(this); allInputSS.addAll((Set<? extends SystemStream>) JavaConverters.setAsJavaSetConverter(taskConfig.getInputStreams()).asJava()); allInputSS.addAll(getBroadcastSystemStreams()); return Collections.unmodifiableSet(allInputSS); }
/** * Get the SystemStreams for the configured input and broadcast streams. * * @return the set of SystemStreams for both standard inputs and broadcast stream inputs. */ public Set<SystemStream> getAllInputStreams() { Set<SystemStream> allInputSS = new HashSet<>(); TaskConfig taskConfig = TaskConfig.Config2Task(this); allInputSS.addAll((Set<? extends SystemStream>) JavaConverters.setAsJavaSetConverter(taskConfig.getInputStreams()).asJava()); allInputSS.addAll(getBroadcastSystemStreams()); return Collections.unmodifiableSet(allInputSS); }
/** * Get the SystemStreams for the configured input and broadcast streams. * * @return the set of SystemStreams for both standard inputs and broadcast stream inputs. */ public Set<SystemStream> getAllInputStreams() { Set<SystemStream> allInputSS = new HashSet<>(); TaskConfig taskConfig = TaskConfig.Config2Task(this); allInputSS.addAll((Set<? extends SystemStream>) JavaConverters.setAsJavaSetConverter(taskConfig.getInputStreams()).asJava()); allInputSS.addAll(getBroadcastSystemStreams()); return Collections.unmodifiableSet(allInputSS); }
/** * Get the SystemStreams for the configured input and broadcast streams. * * @return the set of SystemStreams for both standard inputs and broadcast stream inputs. */ public Set<SystemStream> getAllInputStreams() { Set<SystemStream> allInputSS = new HashSet<>(); TaskConfig taskConfig = TaskConfig.Config2Task(this); allInputSS.addAll((Set<? extends SystemStream>) JavaConverters.setAsJavaSetConverter(taskConfig.getInputStreams()).asJava()); allInputSS.addAll(getBroadcastSystemStreams()); return Collections.unmodifiableSet(allInputSS); }
/** * Get the SystemStreams for the configured input and broadcast streams. * * @return the set of SystemStreams for both standard inputs and broadcast stream inputs. */ public Set<SystemStream> getAllInputStreams() { Set<SystemStream> allInputSS = new HashSet<>(); TaskConfig taskConfig = TaskConfig.Config2Task(this); allInputSS.addAll((Set<? extends SystemStream>) JavaConverters.setAsJavaSetConverter(taskConfig.getInputStreams()).asJava()); allInputSS.addAll(getBroadcastSystemStreams()); return Collections.unmodifiableSet(allInputSS); }
@Override public SystemStreamPartitionGrouper getSystemStreamPartitionGrouper(Config config) { if (!(new TaskConfigJava(config).getBroadcastSystemStreams().isEmpty())) { throw new ConfigException("The job configured with AllSspToSingleTaskGrouper cannot have broadcast streams."); } String processors = config.get(JobConfig.PROCESSOR_LIST()); List<String> processorList = Arrays.asList(processors.split(",")); if (processorList.isEmpty()) { throw new SamzaException("processor list cannot be empty!"); } return new AllSspToSingleTaskGrouper(processorList); } }
@Override public SystemStreamPartitionGrouper getSystemStreamPartitionGrouper(Config config) { if (!(new TaskConfigJava(config).getBroadcastSystemStreams().isEmpty())) { throw new ConfigException("The job configured with AllSspToSingleTaskGrouper cannot have broadcast streams."); } String processors = config.get(JobConfig.PROCESSOR_LIST()); List<String> processorList = Arrays.asList(processors.split(",")); if (processorList.isEmpty()) { throw new SamzaException("processor list cannot be empty!"); } return new AllSspToSingleTaskGrouper(processorList); } }
@Override public SystemStreamPartitionGrouper getSystemStreamPartitionGrouper(Config config) { if (!(new TaskConfigJava(config).getBroadcastSystemStreams().isEmpty())) { throw new ConfigException("The job configured with AllSspToSingleTaskGrouper cannot have broadcast streams."); } String processors = config.get(JobConfig.PROCESSOR_LIST()); List<String> processorList = Arrays.asList(processors.split(",")); if (processorList.isEmpty()) { throw new SamzaException("processor list cannot be empty!"); } return new AllSspToSingleTaskGrouper(processorList); } }
@Override public SystemStreamPartitionGrouper getSystemStreamPartitionGrouper(Config config) { if (!(new TaskConfigJava(config).getBroadcastSystemStreams().isEmpty())) { throw new ConfigException("The job configured with AllSspToSingleTaskGrouper cannot have broadcast streams."); } String processors = config.get(JobConfig.PROCESSOR_LIST()); List<String> processorList = Arrays.asList(processors.split(",")); if (processorList.isEmpty()) { throw new SamzaException("processor list cannot be empty!"); } return new AllSspToSingleTaskGrouper(processorList); } }
@Override public SystemStreamPartitionGrouper getSystemStreamPartitionGrouper(Config config) { if (!(new TaskConfigJava(config).getBroadcastSystemStreams().isEmpty())) { throw new ConfigException("The job configured with AllSspToSingleTaskGrouper cannot have broadcast streams."); } String processors = config.get(JobConfig.PROCESSOR_LIST()); List<String> processorList = Arrays.asList(processors.split(",")); if (processorList.isEmpty()) { throw new SamzaException("processor list cannot be empty!"); } return new AllSspToSingleTaskGrouper(processorList); } }
protected void validateConfig(String system, Config config) { // Kinesis system does not support groupers other than AllSspToSingleTaskGrouper JobConfig jobConfig = new JobConfig(config); if (!jobConfig.getSystemStreamPartitionGrouperFactory().equals( AllSspToSingleTaskGrouperFactory.class.getCanonicalName())) { String errMsg = String.format("Incorrect Grouper %s used for KinesisSystemConsumer %s. Please set the %s config" + " to %s.", jobConfig.getSystemStreamPartitionGrouperFactory(), system, JobConfig.SSP_GROUPER_FACTORY(), AllSspToSingleTaskGrouperFactory.class.getCanonicalName()); throw new ConfigException(errMsg); } // Kinesis streams cannot be configured as broadcast streams TaskConfigJava taskConfig = new TaskConfigJava(config); if (taskConfig.getBroadcastSystemStreams().stream().anyMatch(ss -> system.equals(ss.getSystem()))) { throw new ConfigException("Kinesis streams cannot be configured as broadcast streams."); } // Kinesis streams cannot be configured as bootstrap streams KinesisConfig kConfig = new KinesisConfig(config); kConfig.getKinesisStreams(system).forEach(stream -> { StreamConfig streamConfig = new StreamConfig(kConfig); SystemStream ss = new SystemStream(system, stream); if (streamConfig.getBootstrapEnabled(ss)) { throw new ConfigException("Kinesis streams cannot be configured as bootstrap streams."); } }); } }