@Override public Map<String, String> toConfig() { Map<String, String> config = new HashMap<>(super.toConfig()); String systemName = getSystemName(); this.region.ifPresent( val -> config.put(String.format(KinesisConfig.CONFIG_SYSTEM_REGION, systemName), val)); this.proxyHost.ifPresent( val -> config.put(String.format(KinesisConfig.CONFIG_PROXY_HOST, systemName), val)); this.proxyPort.ifPresent( val -> config.put(String.format(KinesisConfig.CONFIG_PROXY_PORT, systemName), String.valueOf(val))); final String kclConfigPrefix = String.format(KinesisConfig.CONFIG_SYSTEM_KINESIS_CLIENT_LIB_CONFIG, systemName); this.kclConfig.forEach((k, v) -> config.put(kclConfigPrefix + k, v)); final String awsConfigPrefix = String.format(KinesisConfig.CONFIG_AWS_CLIENT_CONFIG, systemName); this.awsConfig.forEach((k, v) -> config.put(awsConfigPrefix + k, v)); return config; } }
public String getSystemName() { return this.systemDescriptor.getSystemName(); }
/** * Constructs an {@link InputDescriptor} instance. * * @param streamId id of the stream * @param serde serde for messages in the stream * @param systemDescriptor system descriptor this stream descriptor was obtained from * @param transformer stream level input stream transform function if available, else null */ public InputDescriptor(String streamId, Serde serde, SystemDescriptor systemDescriptor, InputTransformer transformer) { super(streamId, serde, systemDescriptor); // stream level transformer takes precedence over system level transformer if (transformer != null) { this.transformerOptional = Optional.of(transformer); } else { this.transformerOptional = systemDescriptor.getTransformer(); } }
private Map<String, String> generateSystemStreamConfigs(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { Map<String, String> systemStreamConfigs = new HashMap<>(); appDesc.getInputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getOutputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getSystemDescriptors().forEach(sd -> systemStreamConfigs.putAll(sd.toConfig())); appDesc.getDefaultSystemDescriptor().ifPresent(dsd -> systemStreamConfigs.put(JobConfig.JOB_DEFAULT_SYSTEM(), dsd.getSystemName())); return systemStreamConfigs; } }
@Override public <M> MessageStream<M> getInputStream(InputDescriptor<M, ?> inputDescriptor) { SystemDescriptor systemDescriptor = inputDescriptor.getSystemDescriptor(); Optional<StreamExpander> expander = systemDescriptor.getExpander(); if (expander.isPresent()) { return expander.get().apply(this, inputDescriptor); } // TODO: SAMZA-1841: need to add to the broadcast streams if inputDescriptor is for a broadcast stream addInputDescriptor(inputDescriptor); String streamId = inputDescriptor.getStreamId(); Serde serde = inputDescriptor.getSerde(); KV<Serde, Serde> kvSerdes = getOrCreateStreamSerdes(streamId, serde); boolean isKeyed = serde instanceof KVSerde; InputTransformer transformer = inputDescriptor.getTransformer().orElse(null); InputOperatorSpec inputOperatorSpec = OperatorSpecs.createInputOperatorSpec(streamId, kvSerdes.getKey(), kvSerdes.getValue(), transformer, isKeyed, this.getNextOpId(OpCode.INPUT, null)); inputOperators.put(streamId, inputOperatorSpec); return new MessageStreamImpl(this, inputOperators.get(streamId)); }
/** * Constructs a {@link SystemDescriptor} instance. * * @param systemName name of this system * @param factoryClassName name of the SystemFactory class for this system * @param transformer the {@link InputTransformer} for the system if any, else null * @param expander the {@link StreamExpander} for the system if any, else null */ public SystemDescriptor(String systemName, String factoryClassName, InputTransformer transformer, StreamExpander expander) { Preconditions.checkArgument(isValidSystemName(systemName), String.format("systemName: %s must be non-empty and must not contain spaces or special characters.", systemName)); if (StringUtils.isBlank(factoryClassName)) { LOGGER.warn("Blank SystemFactory class name for system: {}. A value must be provided in configuration using {}.", systemName, String.format(FACTORY_CONFIG_KEY, systemName)); } this.systemName = systemName; this.factoryClassNameOptional = Optional.ofNullable(StringUtils.stripToNull(factoryClassName)); this.transformerOptional = Optional.ofNullable(transformer); this.expanderOptional = Optional.ofNullable(expander); }
private Map<String, String> generateSystemStreamConfigs(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { Map<String, String> systemStreamConfigs = new HashMap<>(); appDesc.getInputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getOutputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getSystemDescriptors().forEach(sd -> systemStreamConfigs.putAll(sd.toConfig())); appDesc.getDefaultSystemDescriptor().ifPresent(dsd -> systemStreamConfigs.put(JobConfig.JOB_DEFAULT_SYSTEM(), dsd.getSystemName())); return systemStreamConfigs; } }
@Override public <M> MessageStream<M> getInputStream(InputDescriptor<M, ?> inputDescriptor) { SystemDescriptor systemDescriptor = inputDescriptor.getSystemDescriptor(); Optional<StreamExpander> expander = systemDescriptor.getExpander(); if (expander.isPresent()) { return expander.get().apply(this, inputDescriptor); } // TODO: SAMZA-1841: need to add to the broadcast streams if inputDescriptor is for a broadcast stream addInputDescriptor(inputDescriptor); String streamId = inputDescriptor.getStreamId(); Serde serde = inputDescriptor.getSerde(); KV<Serde, Serde> kvSerdes = getOrCreateStreamSerdes(streamId, serde); boolean isKeyed = serde instanceof KVSerde; InputTransformer transformer = inputDescriptor.getTransformer().orElse(null); InputOperatorSpec inputOperatorSpec = OperatorSpecs.createInputOperatorSpec(streamId, kvSerdes.getKey(), kvSerdes.getValue(), transformer, isKeyed, this.getNextOpId(OpCode.INPUT, null)); inputOperators.put(streamId, inputOperatorSpec); return new MessageStreamImpl(this, inputOperators.get(streamId)); }
/** * Constructs a {@link SystemDescriptor} instance. * * @param systemName name of this system * @param factoryClassName name of the SystemFactory class for this system * @param transformer the {@link InputTransformer} for the system if any, else null * @param expander the {@link StreamExpander} for the system if any, else null */ public SystemDescriptor(String systemName, String factoryClassName, InputTransformer transformer, StreamExpander expander) { Preconditions.checkArgument(isValidSystemName(systemName), String.format("systemName: %s must be non-empty and must not contain spaces or special characters.", systemName)); if (StringUtils.isBlank(factoryClassName)) { LOGGER.warn("Blank SystemFactory class name for system: {}. A value must be provided in configuration using {}.", systemName, String.format(FACTORY_CONFIG_KEY, systemName)); } this.systemName = systemName; this.factoryClassNameOptional = Optional.ofNullable(StringUtils.stripToNull(factoryClassName)); this.transformerOptional = Optional.ofNullable(transformer); this.expanderOptional = Optional.ofNullable(expander); }
private Map<String, String> generateSystemStreamConfigs(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { Map<String, String> systemStreamConfigs = new HashMap<>(); appDesc.getInputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getOutputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getSystemDescriptors().forEach(sd -> systemStreamConfigs.putAll(sd.toConfig())); appDesc.getDefaultSystemDescriptor().ifPresent(dsd -> systemStreamConfigs.put(JobConfig.JOB_DEFAULT_SYSTEM(), dsd.getSystemName())); return systemStreamConfigs; } }
@Override public Map<String, String> toConfig() { Map<String, String> ehConfigs = new HashMap<>(super.toConfig()); String systemName = getSystemName(); if (!this.streamIds.isEmpty()) { ehConfigs.put(String.format(EventHubConfig.CONFIG_STREAM_LIST, systemName), String.join(",", this.streamIds)); } this.fetchRuntimeInfoTimeout.ifPresent(timeout -> ehConfigs.put(String.format(EventHubConfig.CONFIG_FETCH_RUNTIME_INFO_TIMEOUT_MILLIS, systemName), Integer.toString(timeout))); this.numClientThreads.ifPresent(numClientThreads -> ehConfigs.put(String.format(EventHubConfig.CONFIG_SYSTEM_NUM_CLIENT_THREADS, systemName), Integer.toString(numClientThreads))); this.consumerReceiveQueueSize.ifPresent(receiveQueueSize -> ehConfigs.put(String.format(EventHubConfig.CONFIG_CONSUMER_BUFFER_CAPACITY, systemName), Integer.toString(receiveQueueSize))); this.consumerMaxEventCountPerPoll.ifPresent(count -> ehConfigs.put(String.format(EventHubConfig.CONFIG_MAX_EVENT_COUNT_PER_POLL, systemName), Integer.toString(count))); this.consumerPrefetchCount.ifPresent(count -> ehConfigs.put(String.format(EventHubConfig.CONFIG_PREFETCH_COUNT, systemName), Integer.toString(count))); this.producerEventhubsSendKey.ifPresent(sendKeys -> ehConfigs.put(String.format(EventHubConfig.CONFIG_SEND_KEY_IN_EVENT_PROPERTIES, systemName), Boolean.toString(sendKeys))); this.producerEventhubsPartitioningMethod.ifPresent(partitioningMethod -> ehConfigs.put(String.format(EventHubConfig.CONFIG_PRODUCER_PARTITION_METHOD, systemName), partitioningMethod.toString())); return ehConfigs; } }
public String getSystemName() { return this.systemDescriptor.getSystemName(); }
@Override public <M> MessageStream<M> getInputStream(InputDescriptor<M, ?> inputDescriptor) { SystemDescriptor systemDescriptor = inputDescriptor.getSystemDescriptor(); Optional<StreamExpander> expander = systemDescriptor.getExpander(); if (expander.isPresent()) { return expander.get().apply(this, inputDescriptor); } // TODO: SAMZA-1841: need to add to the broadcast streams if inputDescriptor is for a broadcast stream addInputDescriptor(inputDescriptor); String streamId = inputDescriptor.getStreamId(); Serde serde = inputDescriptor.getSerde(); KV<Serde, Serde> kvSerdes = getOrCreateStreamSerdes(streamId, serde); boolean isKeyed = serde instanceof KVSerde; InputTransformer transformer = inputDescriptor.getTransformer().orElse(null); InputOperatorSpec inputOperatorSpec = OperatorSpecs.createInputOperatorSpec(streamId, kvSerdes.getKey(), kvSerdes.getValue(), transformer, isKeyed, this.getNextOpId(OpCode.INPUT, null)); inputOperators.put(streamId, inputOperatorSpec); return new MessageStreamImpl(this, inputOperators.get(streamId)); }
/** * Constructs an {@link InputDescriptor} instance. * * @param streamId id of the stream * @param serde serde for messages in the stream * @param systemDescriptor system descriptor this stream descriptor was obtained from * @param transformer stream level input stream transform function if available, else null */ public InputDescriptor(String streamId, Serde serde, SystemDescriptor systemDescriptor, InputTransformer transformer) { super(streamId, serde, systemDescriptor); // stream level transformer takes precedence over system level transformer if (transformer != null) { this.transformerOptional = Optional.of(transformer); } else { this.transformerOptional = systemDescriptor.getTransformer(); } }
private Map<String, String> generateSystemStreamConfigs(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { Map<String, String> systemStreamConfigs = new HashMap<>(); appDesc.getInputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getOutputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getSystemDescriptors().forEach(sd -> systemStreamConfigs.putAll(sd.toConfig())); appDesc.getDefaultSystemDescriptor().ifPresent(dsd -> systemStreamConfigs.put(JobConfig.JOB_DEFAULT_SYSTEM(), dsd.getSystemName())); return systemStreamConfigs; } }
@Override public Map<String, String> toConfig() { Map<String, String> configs = new HashMap<>(super.toConfig()); if(!consumerZkConnect.isEmpty()) { configs.put(String.format(CONSUMER_ZK_CONNECT_CONFIG_KEY, getSystemName()), String.join(",", consumerZkConnect)); } consumerAutoOffsetResetOptional.ifPresent(consumerAutoOffsetReset -> configs.put(String.format(CONSUMER_AUTO_OFFSET_RESET_CONFIG_KEY, getSystemName()), consumerAutoOffsetReset)); consumerFetchThresholdOptional.ifPresent(consumerFetchThreshold -> configs.put(String.format(CONSUMER_FETCH_THRESHOLD_CONFIG_KEY, getSystemName()), Integer.toString(consumerFetchThreshold))); consumerFetchThresholdBytesOptional.ifPresent(consumerFetchThresholdBytes -> configs.put(String.format(CONSUMER_FETCH_THRESHOLD_BYTES_CONFIG_KEY, getSystemName()), Long.toString(consumerFetchThresholdBytes))); consumerFetchMessageMaxBytesOptional.ifPresent(consumerFetchMessageMaxBytes -> configs.put(String.format(CONSUMER_FETCH_MESSAGE_MAX_BYTES_KEY, getSystemName()), Long.toString(consumerFetchMessageMaxBytes))); consumerConfigs.forEach((key, value) -> configs.put(String.format(CONSUMER_CONFIGS_CONFIG_KEY, getSystemName(), key), value)); if (!producerBootstrapServers.isEmpty()) { configs.put(String.format(PRODUCER_BOOTSTRAP_SERVERS_CONFIG_KEY, getSystemName()), String.join(",", producerBootstrapServers)); } producerConfigs.forEach((key, value) -> configs.put(String.format(PRODUCER_CONFIGS_CONFIG_KEY, getSystemName(), key), value)); return configs; } }
private void addSystemDescriptor(SystemDescriptor systemDescriptor) { String systemName = systemDescriptor.getSystemName(); Preconditions.checkState(!systemDescriptors.containsKey(systemName) || systemDescriptors.get(systemName) == systemDescriptor, "Must not use different system descriptor instances for the same system name: " + systemName); systemDescriptors.put(systemName, systemDescriptor); } }
@Override public <M> MessageStream<M> getInputStream(InputDescriptor<M, ?> inputDescriptor) { SystemDescriptor systemDescriptor = inputDescriptor.getSystemDescriptor(); Optional<StreamExpander> expander = systemDescriptor.getExpander(); if (expander.isPresent()) { return expander.get().apply(this, inputDescriptor); } // TODO: SAMZA-1841: need to add to the broadcast streams if inputDescriptor is for a broadcast stream addInputDescriptor(inputDescriptor); String streamId = inputDescriptor.getStreamId(); Serde serde = inputDescriptor.getSerde(); KV<Serde, Serde> kvSerdes = getOrCreateStreamSerdes(streamId, serde); boolean isKeyed = serde instanceof KVSerde; InputTransformer transformer = inputDescriptor.getTransformer().orElse(null); InputOperatorSpec inputOperatorSpec = OperatorSpecs.createInputOperatorSpec(streamId, kvSerdes.getKey(), kvSerdes.getValue(), transformer, isKeyed, this.getNextOpId(OpCode.INPUT, null)); inputOperators.put(streamId, inputOperatorSpec); return new MessageStreamImpl(this, inputOperators.get(streamId)); }
private Map<String, String> generateSystemStreamConfigs(ApplicationDescriptorImpl<? extends ApplicationDescriptor> appDesc) { Map<String, String> systemStreamConfigs = new HashMap<>(); appDesc.getInputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getOutputDescriptors().forEach((key, value) -> systemStreamConfigs.putAll(value.toConfig())); appDesc.getSystemDescriptors().forEach(sd -> systemStreamConfigs.putAll(sd.toConfig())); appDesc.getDefaultSystemDescriptor().ifPresent(dsd -> systemStreamConfigs.put(JobConfig.JOB_DEFAULT_SYSTEM(), dsd.getSystemName())); return systemStreamConfigs; } }
@Override public Map<String, String> toConfig() { Map<String, String> configs = new HashMap<>(super.toConfig()); if(!consumerZkConnect.isEmpty()) { configs.put(String.format(CONSUMER_ZK_CONNECT_CONFIG_KEY, getSystemName()), String.join(",", consumerZkConnect)); } consumerAutoOffsetResetOptional.ifPresent(consumerAutoOffsetReset -> configs.put(String.format(CONSUMER_AUTO_OFFSET_RESET_CONFIG_KEY, getSystemName()), consumerAutoOffsetReset)); consumerFetchThresholdOptional.ifPresent(consumerFetchThreshold -> configs.put(String.format(CONSUMER_FETCH_THRESHOLD_CONFIG_KEY, getSystemName()), Integer.toString(consumerFetchThreshold))); consumerFetchThresholdBytesOptional.ifPresent(consumerFetchThresholdBytes -> configs.put(String.format(CONSUMER_FETCH_THRESHOLD_BYTES_CONFIG_KEY, getSystemName()), Long.toString(consumerFetchThresholdBytes))); consumerFetchMessageMaxBytesOptional.ifPresent(consumerFetchMessageMaxBytes -> configs.put(String.format(CONSUMER_FETCH_MESSAGE_MAX_BYTES_KEY, getSystemName()), Long.toString(consumerFetchMessageMaxBytes))); consumerConfigs.forEach((key, value) -> configs.put(String.format(CONSUMER_CONFIGS_CONFIG_KEY, getSystemName(), key), value)); if (!producerBootstrapServers.isEmpty()) { configs.put(String.format(PRODUCER_BOOTSTRAP_SERVERS_CONFIG_KEY, getSystemName()), String.join(",", producerBootstrapServers)); } producerConfigs.forEach((key, value) -> configs.put(String.format(PRODUCER_CONFIGS_CONFIG_KEY, getSystemName(), key), value)); return configs; } }