@Override public Map<String, String> toConfig() { HashMap<String, String> ehConfigs = new HashMap<>(super.toConfig()); String streamId = getStreamId(); ehConfigs.put(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamId), namespace); ehConfigs.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamId), entityPath); sasKeyName.ifPresent(keyName -> ehConfigs.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamId), keyName)); sasToken.ifPresent(key -> ehConfigs.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamId), key)); this.consumerGroup.ifPresent(consumerGroupName -> ehConfigs.put(String.format(EventHubConfig.CONFIG_STREAM_CONSUMER_GROUP, streamId), consumerGroupName)); return ehConfigs; }
@Override public Map<String, String> toConfig() { Map<String, String> config = new HashMap<>(super.toConfig()); String systemName = getSystemName(); String streamId = getStreamId(); String clientConfigPrefix = String.format(KinesisConfig.CONFIG_STREAM_KINESIS_CLIENT_LIB_CONFIG, systemName, streamId); this.region.ifPresent( val -> config.put(String.format(KinesisConfig.CONFIG_STREAM_REGION, systemName, streamId), val)); this.accessKey.ifPresent( val -> config.put(String.format(KinesisConfig.CONFIG_STREAM_ACCESS_KEY, systemName, streamId), val)); this.secretKey.ifPresent( val -> config.put(String.format(KinesisConfig.CONFIG_STREAM_SECRET_KEY, systemName, streamId), val)); this.kclConfig.forEach((k, v) -> config.put(clientConfigPrefix + k, v)); return config; } }
@Override public Map<String, String> toConfig() { HashMap<String, String> configs = new HashMap<>(super.toConfig()); // Note: Kafka configuration needs the topic's physical name, not the stream-id. // We won't have that here if user only specified it in configs, or if it got rewritten // by the planner to something different than what's in this descriptor. String streamName = getPhysicalName().orElse(getStreamId()); String systemName = getSystemName(); consumerAutoOffsetResetOptional.ifPresent(autoOffsetReset -> configs.put(String.format(CONSUMER_AUTO_OFFSET_RESET_CONFIG_KEY, systemName, streamName), autoOffsetReset)); consumerFetchMessageMaxBytesOptional.ifPresent(fetchMessageMaxBytes -> configs.put(String.format(CONSUMER_FETCH_MESSAGE_MAX_BYTES_CONFIG_KEY, systemName, streamName), Long.toString(fetchMessageMaxBytes))); return configs; } }
@Override public Map<String, String> toConfig() { HashMap<String, String> configs = new HashMap<>(super.toConfig()); // Note: Kafka configuration needs the topic's physical name, not the stream-id. // We won't have that here if user only specified it in configs, or if it got rewritten // by the planner to something different than what's in this descriptor. String streamName = getPhysicalName().orElse(getStreamId()); String systemName = getSystemName(); consumerAutoOffsetResetOptional.ifPresent(autoOffsetReset -> configs.put(String.format(CONSUMER_AUTO_OFFSET_RESET_CONFIG_KEY, systemName, streamName), autoOffsetReset)); consumerFetchMessageMaxBytesOptional.ifPresent(fetchMessageMaxBytes -> configs.put(String.format(CONSUMER_FETCH_MESSAGE_MAX_BYTES_CONFIG_KEY, systemName, streamName), Long.toString(fetchMessageMaxBytes))); return configs; } }
@Override public Map<String, String> toConfig() { HashMap<String, String> configs = new HashMap<>(super.toConfig()); // Note: Kafka configuration needs the topic's physical name, not the stream-id. // We won't have that here if user only specified it in configs, or if it got rewritten // by the planner to something different than what's in this descriptor. String streamName = getPhysicalName().orElse(getStreamId()); String systemName = getSystemName(); consumerAutoOffsetResetOptional.ifPresent(autoOffsetReset -> configs.put(String.format(CONSUMER_AUTO_OFFSET_RESET_CONFIG_KEY, systemName, streamName), autoOffsetReset)); consumerFetchMessageMaxBytesOptional.ifPresent(fetchMessageMaxBytes -> configs.put(String.format(CONSUMER_FETCH_MESSAGE_MAX_BYTES_CONFIG_KEY, systemName, streamName), Long.toString(fetchMessageMaxBytes))); return configs; } }
private Map<String, String> generateSystemStreamConfigs(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { Map<String, String> systemStreamConfigs = new HashMap<>(); appDesc.getInputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getOutputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getSystemDescriptors().forEach(sd -> systemStreamConfigs.putAll(sd.toConfig())); appDesc.getDefaultSystemDescriptor().ifPresent(dsd -> systemStreamConfigs.put(JobConfig.JOB_DEFAULT_SYSTEM(), dsd.getSystemName())); return systemStreamConfigs; } }
private Map<String, String> generateSystemStreamConfigs(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { Map<String, String> systemStreamConfigs = new HashMap<>(); appDesc.getInputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getOutputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getSystemDescriptors().forEach(sd -> systemStreamConfigs.putAll(sd.toConfig())); appDesc.getDefaultSystemDescriptor().ifPresent(dsd -> systemStreamConfigs.put(JobConfig.JOB_DEFAULT_SYSTEM(), dsd.getSystemName())); return systemStreamConfigs; } }
private Map<String, String> generateSystemStreamConfigs(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { Map<String, String> systemStreamConfigs = new HashMap<>(); appDesc.getInputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getOutputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getSystemDescriptors().forEach(sd -> systemStreamConfigs.putAll(sd.toConfig())); appDesc.getDefaultSystemDescriptor().ifPresent(dsd -> systemStreamConfigs.put(JobConfig.JOB_DEFAULT_SYSTEM(), dsd.getSystemName())); return systemStreamConfigs; } }
private Map<String, String> generateSystemStreamConfigs(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { Map<String, String> systemStreamConfigs = new HashMap<>(); appDesc.getInputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getOutputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getSystemDescriptors().forEach(sd -> systemStreamConfigs.putAll(sd.toConfig())); appDesc.getDefaultSystemDescriptor().ifPresent(dsd -> systemStreamConfigs.put(JobConfig.JOB_DEFAULT_SYSTEM(), dsd.getSystemName())); return systemStreamConfigs; } }
private Map<String, String> generateSystemStreamConfigs(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { Map<String, String> systemStreamConfigs = new HashMap<>(); appDesc.getInputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getOutputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getSystemDescriptors().forEach(sd -> systemStreamConfigs.putAll(sd.toConfig())); appDesc.getDefaultSystemDescriptor().ifPresent(dsd -> systemStreamConfigs.put(JobConfig.JOB_DEFAULT_SYSTEM(), dsd.getSystemName())); return systemStreamConfigs; } }
@Test public void testCreateJobGraphForLegacyTaskApplication() { TaskApplicationDescriptorImpl taskAppDesc = mock(TaskApplicationDescriptorImpl.class); when(taskAppDesc.getInputDescriptors()).thenReturn(new HashMap<>()); when(taskAppDesc.getOutputDescriptors()).thenReturn(new HashMap<>()); when(taskAppDesc.getTableDescriptors()).thenReturn(new HashSet<>()); when(taskAppDesc.getSystemDescriptors()).thenReturn(new HashSet<>()); when(taskAppDesc.getIntermediateBroadcastStreamIds()).thenReturn(new HashSet<>()); doReturn(LegacyTaskApplication.class).when(taskAppDesc).getAppClass(); Map<String, String> systemStreamConfigs = new HashMap<>(); inputDescriptors.forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); outputDescriptors.forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); systemDescriptors.forEach(sd -> systemStreamConfigs.putAll(sd.toConfig())); ExecutionPlanner planner = new ExecutionPlanner(config, streamManager); JobGraph jobGraph = planner.createJobGraph(taskAppDesc); assertEquals(1, jobGraph.getJobNodes().size()); JobNode jobNode = jobGraph.getJobNodes().get(0); assertEquals("test-app", jobNode.getJobName()); assertEquals("test-app-1", jobNode.getJobNameAndId()); assertEquals(0, jobNode.getInEdges().size()); assertEquals(0, jobNode.getOutEdges().size()); assertEquals(0, jobNode.getTables().size()); assertEquals(config, jobNode.getConfig()); }
inputDescriptors.forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); outputDescriptors.forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); systemDescriptors.forEach(sd -> systemStreamConfigs.putAll(sd.toConfig()));