@Override public Map<String, String> toConfig() { HashMap<String, String> ehConfigs = new HashMap<>(super.toConfig()); String streamId = getStreamId(); ehConfigs.put(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamId), namespace); ehConfigs.put(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamId), entityPath); sasKeyName.ifPresent(keyName -> ehConfigs.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamId), keyName)); sasToken.ifPresent(key -> ehConfigs.put(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamId), key)); this.consumerGroup.ifPresent(consumerGroupName -> ehConfigs.put(String.format(EventHubConfig.CONFIG_STREAM_CONSUMER_GROUP, streamId), consumerGroupName)); return ehConfigs; }
final void addInputDescriptor(InputDescriptor inputDescriptor) { String streamId = inputDescriptor.getStreamId(); Preconditions.checkState(!inputDescriptors.containsKey(streamId) || inputDescriptors.get(streamId) == inputDescriptor, String.format("Cannot add multiple input descriptors with the same streamId: %s", streamId)); inputDescriptors.put(streamId, inputDescriptor); addSystemDescriptor(inputDescriptor.getSystemDescriptor()); }
@Override public TaskApplicationDescriptor withInputStream(InputDescriptor inputDescriptor) { // TODO: SAMZA-1841: need to add to the broadcast streams if inputDescriptor is for a broadcast stream addInputDescriptor(inputDescriptor); getOrCreateStreamSerdes(inputDescriptor.getStreamId(), inputDescriptor.getSerde()); return this; }
@Override public <M> MessageStream<M> getInputStream(InputDescriptor<M, ?> inputDescriptor) { SystemDescriptor systemDescriptor = inputDescriptor.getSystemDescriptor(); Optional<StreamExpander> expander = systemDescriptor.getExpander(); if (expander.isPresent()) { return expander.get().apply(this, inputDescriptor); } // TODO: SAMZA-1841: need to add to the broadcast streams if inputDescriptor is for a broadcast stream addInputDescriptor(inputDescriptor); String streamId = inputDescriptor.getStreamId(); Serde serde = inputDescriptor.getSerde(); KV<Serde, Serde> kvSerdes = getOrCreateStreamSerdes(streamId, serde); boolean isKeyed = serde instanceof KVSerde; InputTransformer transformer = inputDescriptor.getTransformer().orElse(null); InputOperatorSpec inputOperatorSpec = OperatorSpecs.createInputOperatorSpec(streamId, kvSerdes.getKey(), kvSerdes.getValue(), transformer, isKeyed, this.getNextOpId(OpCode.INPUT, null)); inputOperators.put(streamId, inputOperatorSpec); return new MessageStreamImpl(this, inputOperators.get(streamId)); }
@Override public Map<String, String> toConfig() { HashMap<String, String> configs = new HashMap<>(super.toConfig()); String streamId = getStreamId(); this.offsetDefaultOptional.ifPresent(od -> configs.put(String.format(OFFSET_DEFAULT_CONFIG_KEY, streamId), od.name().toLowerCase())); this.resetOffsetOptional.ifPresent(resetOffset -> configs.put(String.format(RESET_OFFSET_CONFIG_KEY, streamId), Boolean.toString(resetOffset))); this.priorityOptional.ifPresent(priority -> configs.put(String.format(PRIORITY_CONFIG_KEY, streamId), Integer.toString(priority))); this.isBootstrapOptional.ifPresent(bootstrap -> configs.put(String.format(BOOTSTRAP_CONFIG_KEY, streamId), Boolean.toString(bootstrap))); this.isBoundedOptional.ifPresent(bounded -> configs.put(String.format(BOUNDED_CONFIG_KEY, streamId), Boolean.toString(bounded))); this.deleteCommittedMessagesOptional.ifPresent(deleteCommittedMessages -> configs.put(String.format(DELETE_COMMITTED_MESSAGES_CONFIG_KEY, streamId), Boolean.toString(deleteCommittedMessages))); return Collections.unmodifiableMap(configs); } }
@Before public void setUp() { when(defaultSystemDescriptor.getSystemName()).thenReturn(defaultSystemName); mockInputs.forEach(isd -> when(isd.getSystemDescriptor()).thenReturn(defaultSystemDescriptor)); mockOutputs.forEach(osd -> when(osd.getSystemDescriptor()).thenReturn(defaultSystemDescriptor)); }
@Override public GenericInputDescriptor<StreamMessageType> withPhysicalName(String physicalName) { return super.withPhysicalName(physicalName); } }
@Override public <M> MessageStream<M> getInputStream(InputDescriptor<M, ?> inputDescriptor) { SystemDescriptor systemDescriptor = inputDescriptor.getSystemDescriptor(); Optional<StreamExpander> expander = systemDescriptor.getExpander(); if (expander.isPresent()) { return expander.get().apply(this, inputDescriptor); } // TODO: SAMZA-1841: need to add to the broadcast streams if inputDescriptor is for a broadcast stream addInputDescriptor(inputDescriptor); String streamId = inputDescriptor.getStreamId(); Serde serde = inputDescriptor.getSerde(); KV<Serde, Serde> kvSerdes = getOrCreateStreamSerdes(streamId, serde); boolean isKeyed = serde instanceof KVSerde; InputTransformer transformer = inputDescriptor.getTransformer().orElse(null); InputOperatorSpec inputOperatorSpec = OperatorSpecs.createInputOperatorSpec(streamId, kvSerdes.getKey(), kvSerdes.getValue(), transformer, isKeyed, this.getNextOpId(OpCode.INPUT, null)); inputOperators.put(streamId, inputOperatorSpec); return new MessageStreamImpl(this, inputOperators.get(streamId)); }
@Override public Map<String, String> toConfig() { HashMap<String, String> configs = new HashMap<>(super.toConfig()); String streamId = getStreamId(); this.offsetDefaultOptional.ifPresent(od -> configs.put(String.format(OFFSET_DEFAULT_CONFIG_KEY, streamId), od.name().toLowerCase())); this.resetOffsetOptional.ifPresent(resetOffset -> configs.put(String.format(RESET_OFFSET_CONFIG_KEY, streamId), Boolean.toString(resetOffset))); this.priorityOptional.ifPresent(priority -> configs.put(String.format(PRIORITY_CONFIG_KEY, streamId), Integer.toString(priority))); this.isBootstrapOptional.ifPresent(bootstrap -> configs.put(String.format(BOOTSTRAP_CONFIG_KEY, streamId), Boolean.toString(bootstrap))); this.isBoundedOptional.ifPresent(bounded -> configs.put(String.format(BOUNDED_CONFIG_KEY, streamId), Boolean.toString(bounded))); this.deleteCommittedMessagesOptional.ifPresent(deleteCommittedMessages -> configs.put(String.format(DELETE_COMMITTED_MESSAGES_CONFIG_KEY, streamId), Boolean.toString(deleteCommittedMessages))); return Collections.unmodifiableMap(configs); } }
@Override public GenericInputDescriptor<StreamMessageType> withPhysicalName(String physicalName) { return super.withPhysicalName(physicalName); } }
@Override public <M> MessageStream<M> getInputStream(InputDescriptor<M, ?> inputDescriptor) { SystemDescriptor systemDescriptor = inputDescriptor.getSystemDescriptor(); Optional<StreamExpander> expander = systemDescriptor.getExpander(); if (expander.isPresent()) { return expander.get().apply(this, inputDescriptor); } // TODO: SAMZA-1841: need to add to the broadcast streams if inputDescriptor is for a broadcast stream addInputDescriptor(inputDescriptor); String streamId = inputDescriptor.getStreamId(); Serde serde = inputDescriptor.getSerde(); KV<Serde, Serde> kvSerdes = getOrCreateStreamSerdes(streamId, serde); boolean isKeyed = serde instanceof KVSerde; InputTransformer transformer = inputDescriptor.getTransformer().orElse(null); InputOperatorSpec inputOperatorSpec = OperatorSpecs.createInputOperatorSpec(streamId, kvSerdes.getKey(), kvSerdes.getValue(), transformer, isKeyed, this.getNextOpId(OpCode.INPUT, null)); inputOperators.put(streamId, inputOperatorSpec); return new MessageStreamImpl(this, inputOperators.get(streamId)); }
@Override public Map<String, String> toConfig() { Map<String, String> config = new HashMap<>(super.toConfig()); String systemName = getSystemName(); String streamId = getStreamId(); String clientConfigPrefix = String.format(KinesisConfig.CONFIG_STREAM_KINESIS_CLIENT_LIB_CONFIG, systemName, streamId); this.region.ifPresent( val -> config.put(String.format(KinesisConfig.CONFIG_STREAM_REGION, systemName, streamId), val)); this.accessKey.ifPresent( val -> config.put(String.format(KinesisConfig.CONFIG_STREAM_ACCESS_KEY, systemName, streamId), val)); this.secretKey.ifPresent( val -> config.put(String.format(KinesisConfig.CONFIG_STREAM_SECRET_KEY, systemName, streamId), val)); this.kclConfig.forEach((k, v) -> config.put(clientConfigPrefix + k, v)); return config; } }
final void addInputDescriptor(InputDescriptor inputDescriptor) { String streamId = inputDescriptor.getStreamId(); Preconditions.checkState(!inputDescriptors.containsKey(streamId) || inputDescriptors.get(streamId) == inputDescriptor, String.format("Cannot add multiple input descriptors with the same streamId: %s", streamId)); inputDescriptors.put(streamId, inputDescriptor); addSystemDescriptor(inputDescriptor.getSystemDescriptor()); }
@Override public TaskApplicationDescriptor withInputStream(InputDescriptor inputDescriptor) { // TODO: SAMZA-1841: need to add to the broadcast streams if inputDescriptor is for a broadcast stream addInputDescriptor(inputDescriptor); getOrCreateStreamSerdes(inputDescriptor.getStreamId(), inputDescriptor.getSerde()); return this; }
@Override public <M> MessageStream<M> getInputStream(InputDescriptor<M, ?> inputDescriptor) { SystemDescriptor systemDescriptor = inputDescriptor.getSystemDescriptor(); Optional<StreamExpander> expander = systemDescriptor.getExpander(); if (expander.isPresent()) { return expander.get().apply(this, inputDescriptor); } // TODO: SAMZA-1841: need to add to the broadcast streams if inputDescriptor is for a broadcast stream addInputDescriptor(inputDescriptor); String streamId = inputDescriptor.getStreamId(); Serde serde = inputDescriptor.getSerde(); KV<Serde, Serde> kvSerdes = getOrCreateStreamSerdes(streamId, serde); boolean isKeyed = serde instanceof KVSerde; InputTransformer transformer = inputDescriptor.getTransformer().orElse(null); InputOperatorSpec inputOperatorSpec = OperatorSpecs.createInputOperatorSpec(streamId, kvSerdes.getKey(), kvSerdes.getValue(), transformer, isKeyed, this.getNextOpId(OpCode.INPUT, null)); inputOperators.put(streamId, inputOperatorSpec); return new MessageStreamImpl(this, inputOperators.get(streamId)); }
@Override public Map<String, String> toConfig() { HashMap<String, String> configs = new HashMap<>(super.toConfig()); // Note: Kafka configuration needs the topic's physical name, not the stream-id. // We won't have that here if user only specified it in configs, or if it got rewritten // by the planner to something different than what's in this descriptor. String streamName = getPhysicalName().orElse(getStreamId()); String systemName = getSystemName(); consumerAutoOffsetResetOptional.ifPresent(autoOffsetReset -> configs.put(String.format(CONSUMER_AUTO_OFFSET_RESET_CONFIG_KEY, systemName, streamName), autoOffsetReset)); consumerFetchMessageMaxBytesOptional.ifPresent(fetchMessageMaxBytes -> configs.put(String.format(CONSUMER_FETCH_MESSAGE_MAX_BYTES_CONFIG_KEY, systemName, streamName), Long.toString(fetchMessageMaxBytes))); return configs; } }
final void addInputDescriptor(InputDescriptor inputDescriptor) { String streamId = inputDescriptor.getStreamId(); Preconditions.checkState(!inputDescriptors.containsKey(streamId) || inputDescriptors.get(streamId) == inputDescriptor, String.format("Cannot add multiple input descriptors with the same streamId: %s", streamId)); inputDescriptors.put(streamId, inputDescriptor); addSystemDescriptor(inputDescriptor.getSystemDescriptor()); }
@Override public TaskApplicationDescriptor withInputStream(InputDescriptor inputDescriptor) { // TODO: SAMZA-1841: need to add to the broadcast streams if inputDescriptor is for a broadcast stream addInputDescriptor(inputDescriptor); getOrCreateStreamSerdes(inputDescriptor.getStreamId(), inputDescriptor.getSerde()); return this; }
@Override public <M> MessageStream<M> getInputStream(InputDescriptor<M, ?> inputDescriptor) { SystemDescriptor systemDescriptor = inputDescriptor.getSystemDescriptor(); Optional<StreamExpander> expander = systemDescriptor.getExpander(); if (expander.isPresent()) { return expander.get().apply(this, inputDescriptor); } // TODO: SAMZA-1841: need to add to the broadcast streams if inputDescriptor is for a broadcast stream addInputDescriptor(inputDescriptor); String streamId = inputDescriptor.getStreamId(); Serde serde = inputDescriptor.getSerde(); KV<Serde, Serde> kvSerdes = getOrCreateStreamSerdes(streamId, serde); boolean isKeyed = serde instanceof KVSerde; InputTransformer transformer = inputDescriptor.getTransformer().orElse(null); InputOperatorSpec inputOperatorSpec = OperatorSpecs.createInputOperatorSpec(streamId, kvSerdes.getKey(), kvSerdes.getValue(), transformer, isKeyed, this.getNextOpId(OpCode.INPUT, null)); inputOperators.put(streamId, inputOperatorSpec); return new MessageStreamImpl(this, inputOperators.get(streamId)); }
@Override public Map<String, String> toConfig() { HashMap<String, String> configs = new HashMap<>(super.toConfig()); // Note: Kafka configuration needs the topic's physical name, not the stream-id. // We won't have that here if user only specified it in configs, or if it got rewritten // by the planner to something different than what's in this descriptor. String streamName = getPhysicalName().orElse(getStreamId()); String systemName = getSystemName(); consumerAutoOffsetResetOptional.ifPresent(autoOffsetReset -> configs.put(String.format(CONSUMER_AUTO_OFFSET_RESET_CONFIG_KEY, systemName, streamName), autoOffsetReset)); consumerFetchMessageMaxBytesOptional.ifPresent(fetchMessageMaxBytes -> configs.put(String.format(CONSUMER_FETCH_MESSAGE_MAX_BYTES_CONFIG_KEY, systemName, streamName), Long.toString(fetchMessageMaxBytes))); return configs; } }