/** * Gets the {@link SystemStream} corresponding to the provided stream, which may be * a streamId, or stream name of the format systemName.streamName. * * @param config the config for the job * @param stream the stream name or id to get the {@link SystemStream} for. * @return the {@link SystemStream} for the stream */ public static SystemStream getSystemStreamFromNameOrId(Config config, String stream) { String[] parts = stream.split("\\."); if (parts.length == 0 || parts.length > 2) { throw new SamzaException( String.format("Invalid stream %s. Expected to be of the format streamId or systemName.streamName", stream)); } if (parts.length == 1) { return new StreamConfig(config).streamIdToSystemStream(stream); } else { return new SystemStream(parts[0], parts[1]); } }
public static StreamSpec getStreamSpec(String streamId, StreamConfig streamConfig) { String physicalName = streamConfig.getPhysicalName(streamId); String system = streamConfig.getSystem(streamId); Map<String, String> streamProperties = streamConfig.getStreamProperties(streamId); return new StreamSpec(streamId, physicalName, system, streamProperties); } }
Config generateConfig() { Map<String, String> streamConfig = new HashMap<>(); StreamSpec spec = getStreamSpec(); streamConfig.put(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), spec.getId()), spec.getSystemName()); streamConfig.put(String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), spec.getId()), spec.getPhysicalName()); if (isIntermediate()) { streamConfig.put(String.format(StreamConfig.IS_INTERMEDIATE_FOR_STREAM_ID(), spec.getId()), "true"); streamConfig.put(String.format(StreamConfig.DELETE_COMMITTED_MESSAGES_FOR_STREAM_ID(), spec.getId()), "true"); streamConfig.put(String.format(StreamConfig.CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID(), spec.getId()), "oldest"); streamConfig.put(String.format(StreamConfig.PRIORITY_FOR_STREAM_ID(), spec.getId()), String.valueOf(Integer.MAX_VALUE)); } spec.getConfig().forEach((property, value) -> { streamConfig.put(String.format(StreamConfig.STREAM_ID_PREFIX(), spec.getId()) + property, value); }); return new MapConfig(streamConfig); }
public EventHubConfig(Config config) { super(config); // Build reverse index for streamName -> streamId StreamConfig streamConfig = new StreamConfig(config); LOG.info("Building mappings from physicalName to streamId"); JavaConversions.asJavaCollection(streamConfig.getStreamIds()) .forEach((streamId) -> { String physicalName = streamConfig.getPhysicalName(streamId); LOG.info("Obtained physicalName: {} for streamId: {} ", physicalName, streamId); physcialToId.put(physicalName, streamId); }); }
public String getStreamSerdeName(String systemName, String streamName) { StreamConfig streamConfig = new StreamConfig(this); scala.Option<String> option = streamConfig.getStreamMsgSerde(new SystemStream(systemName, streamName)); return option.isEmpty() ? null : option.get(); } }
OperatorSpecGraph specGraph = appDesc.getOperatorSpecGraph(); StreamConfig streamConfig = new StreamConfig(samzaConfig); String inputStreamId1 = specGraph.getInputOperators().keySet().stream().findFirst().get(); String inputSystem1 = streamConfig.getSystem(inputStreamId1); String inputPhysicalName1 = streamConfig.getPhysicalName(inputStreamId1); String inputStreamId2 = specGraph.getInputOperators().keySet().stream().skip(1).findFirst().get(); String inputSystem2 = streamConfig.getSystem(inputStreamId2); String inputPhysicalName2 = streamConfig.getPhysicalName(inputStreamId2); String outputSystem1 = streamConfig.getSystem(outputStreamId1); String outputPhysicalName1 = streamConfig.getPhysicalName(outputStreamId1); String outputStreamId2 = specGraph.getOutputStreams().keySet().stream().skip(1).findFirst().get(); String outputSystem2 = streamConfig.getSystem(outputStreamId2); String outputPhysicalName2 = streamConfig.getPhysicalName(outputStreamId2);
LOGGER.info("run.id from previous run is {}", appConfig.getRunId()); StreamConfig streamConfig = new StreamConfig(prevConfig); Set<StreamSpec> intStreams = JavaConversions.asJavaCollection(streamConfig.getStreamIds()).stream() .filter(streamConfig::getIsIntermediateStream) .map(id -> new StreamSpec(id, streamConfig.getPhysicalName(id), streamConfig.getSystem(id))) .collect(Collectors.toSet()); intStreams.forEach(stream -> {
@Test(expected = IllegalArgumentException.class) public void testGetStreamSystemNameArgNull() { Config config = buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, StreamConfig.SYSTEM(), null); StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); }
public ExecutionPlanner(Config config, StreamManager streamManager) { this.config = config; this.streamManager = streamManager; this.streamConfig = new StreamConfig(config); }
@Test public void testGenerateConfig() { // an example unbounded IO stream StreamSpec spec = new StreamSpec("stream-1", "physical-stream-1", "system-1", Collections.singletonMap("property1", "haha")); StreamEdge edge = new StreamEdge(spec, false, false, new MapConfig()); Config config = edge.generateConfig(); StreamConfig streamConfig = new StreamConfig(config); assertEquals(streamConfig.getSystem(spec.getId()), "system-1"); assertEquals(streamConfig.getPhysicalName(spec.getId()), "physical-stream-1"); assertEquals(streamConfig.getIsIntermediateStream(spec.getId()), false); assertEquals(streamConfig.getStreamProperties(spec.getId()).get("property1"), "haha"); // bounded stream spec = new StreamSpec("stream-1", "physical-stream-1", "system-1", Collections.singletonMap("property1", "haha")); edge = new StreamEdge(spec, false, false, new MapConfig()); config = edge.generateConfig(); streamConfig = new StreamConfig(config); // intermediate stream edge = new StreamEdge(spec, true, false, new MapConfig()); config = edge.generateConfig(); streamConfig = new StreamConfig(config); assertEquals(streamConfig.getIsIntermediateStream(spec.getId()), true); assertEquals(streamConfig.getDefaultStreamOffset(spec.toSystemStream()).get(), "oldest"); assertEquals(streamConfig.getPriority(spec.toSystemStream()), Integer.MAX_VALUE); } }
String streamIdPrefix = String.format(StreamConfig.STREAM_ID_PREFIX(), streamId); String keySerdeConfigKey = streamIdPrefix + StreamConfig.KEY_SERDE(); configs.put(keySerdeConfigKey, serdeUUIDs.get(serde)); }); String streamIdPrefix = String.format(StreamConfig.STREAM_ID_PREFIX(), streamId); String valueSerdeConfigKey = streamIdPrefix + StreamConfig.MSG_SERDE(); configs.put(valueSerdeConfigKey, serdeUUIDs.get(serde)); });
@Test public void testGetStreamSamzaPropertiesOmitted() { Config config = buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME, StreamConfig.SYSTEM(), TEST_SYSTEM, "systemProperty1", "systemValue1", "systemProperty2", "systemValue2", "systemProperty3", "systemValue3"); StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); Map<String, String> properties = spec.getConfig(); assertEquals(3, properties.size()); assertNull(properties.get(String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID))); assertNull(properties.get(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID))); assertNull(spec.get(String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), STREAM_ID))); assertNull(spec.get(String.format(StreamConfig.SYSTEM_FOR_STREAM_ID(), STREAM_ID))); }
@Test public void testGetSerdeStreamsFromSystemStreamAndStreamId() { StreamConfig config = buildConfig(buildProp(SYSTEM_STREAM_1, StreamConfig.KEY_SERDE()), "value1", buildProp(STREAM1_STREAM_ID, StreamConfig.SYSTEM()), STREAM1_SYSTEM, buildProp(STREAM1_STREAM_ID, StreamConfig.PHYSICAL_NAME()), STREAM1_PHYSICAL_NAME, buildProp(STREAM1_STREAM_ID, StreamConfig.KEY_SERDE()), "value1OVERRIDE", buildProp(STREAM2_STREAM_ID, StreamConfig.SYSTEM()), STREAM2_SYSTEM, buildProp(STREAM2_STREAM_ID, StreamConfig.PHYSICAL_NAME()), STREAM2_PHYSICAL_NAME, buildProp(STREAM2_STREAM_ID, StreamConfig.MSG_SERDE()), "value2", "key3", "value3"); assertEquals(1, config.getSerdeStreams(STREAM1_SYSTEM).size()); assertEquals(1, config.getSerdeStreams(STREAM2_SYSTEM).size()); assertEquals("value1OVERRIDE", config.getStreamKeySerde(SYSTEM_STREAM_1).get()); assertEquals("value2", config.getStreamMsgSerde(SYSTEM_STREAM_2).get()); }
@Test(expected = IllegalArgumentException.class) public void testGetStreamStreamIdInvalid() { Config config = buildStreamConfig(STREAM_ID_INVALID, StreamConfig.SYSTEM(), TEST_SYSTEM); StreamUtil.getStreamSpec(STREAM_ID_INVALID, new StreamConfig(config)); }
StreamConfig config = buildConfig( buildSystemDefaultProp(STREAM1_SYSTEM, nonSamzaProperty), "1", buildSystemDefaultProp(STREAM1_SYSTEM, StreamConfig.KEY_SERDE()), "value1", buildSystemDefaultProp(STREAM1_SYSTEM, StreamConfig.CONSUMER_OFFSET_DEFAULT()), "newest", buildProp(SYSTEM_STREAM_1, "dummyStreamProperty"), "dummyValue", buildProp(STREAM1_STREAM_ID, StreamConfig.SYSTEM()), STREAM1_SYSTEM, buildProp(STREAM1_STREAM_ID, StreamConfig.PHYSICAL_NAME()), STREAM1_PHYSICAL_NAME, buildSystemDefaultProp(STREAM2_SYSTEM, nonSamzaProperty), "2", buildProp(STREAM2_STREAM_ID, StreamConfig.SYSTEM()), STREAM2_SYSTEM, buildProp(STREAM2_STREAM_ID, StreamConfig.PHYSICAL_NAME()), STREAM2_PHYSICAL_NAME, buildProp(STREAM2_STREAM_ID, nonSamzaProperty), "3", buildSystemDefaultProp(STREAM3_SYSTEM, nonSamzaProperty), "4", buildProp(STREAM3_STREAM_ID, StreamConfig.SYSTEM()), STREAM3_SYSTEM, buildProp(STREAM3_STREAM_ID, StreamConfig.PHYSICAL_NAME()), STREAM3_PHYSICAL_NAME, buildProp(SYSTEM_STREAM_3, nonSamzaProperty), "5", "key3", "value3"); assertEquals("value1", config.getStreamKeySerde(SYSTEM_STREAM_1).get()); assertEquals(0, config.getSerdeStreams(STREAM1_SYSTEM).size()); assertEquals("value1", new SystemConfig(config).getSystemKeySerde(STREAM1_SYSTEM).get()); assertEquals("newest", config.getDefaultStreamOffset(SYSTEM_STREAM_1).get()); assertEquals("1", config.getStreamProperties(STREAM1_STREAM_ID).get(nonSamzaProperty)); assertEquals("3", config.getStreamProperties(STREAM2_STREAM_ID).get(nonSamzaProperty)); assertEquals("5", config.getStreamProperties(STREAM3_STREAM_ID).get(nonSamzaProperty)); assertEquals(false, config.getBroadcastEnabled(SYSTEM_STREAM_1));
@Test public void testGetSerdeStreamsFromSystemStream() { StreamConfig config = buildConfig(buildProp(SYSTEM_STREAM_1, StreamConfig.KEY_SERDE()), "value1", buildProp(SYSTEM_STREAM_2, StreamConfig.MSG_SERDE()), "value2", "key3", "value3"); assertEquals(1, config.getSerdeStreams(STREAM1_SYSTEM).size()); assertEquals(1, config.getSerdeStreams(STREAM2_SYSTEM).size()); assertEquals("value1", config.getStreamKeySerde(SYSTEM_STREAM_1).get()); assertEquals("value2", config.getStreamMsgSerde(SYSTEM_STREAM_2).get()); }
protected void validateConfig(String system, Config config) { // Kinesis system does not support groupers other than AllSspToSingleTaskGrouper JobConfig jobConfig = new JobConfig(config); if (!jobConfig.getSystemStreamPartitionGrouperFactory().equals( AllSspToSingleTaskGrouperFactory.class.getCanonicalName())) { String errMsg = String.format("Incorrect Grouper %s used for KinesisSystemConsumer %s. Please set the %s config" + " to %s.", jobConfig.getSystemStreamPartitionGrouperFactory(), system, JobConfig.SSP_GROUPER_FACTORY(), AllSspToSingleTaskGrouperFactory.class.getCanonicalName()); throw new ConfigException(errMsg); } // Kinesis streams cannot be configured as broadcast streams TaskConfigJava taskConfig = new TaskConfigJava(config); if (taskConfig.getBroadcastSystemStreams().stream().anyMatch(ss -> system.equals(ss.getSystem()))) { throw new ConfigException("Kinesis streams cannot be configured as broadcast streams."); } // Kinesis streams cannot be configured as bootstrap streams KinesisConfig kConfig = new KinesisConfig(config); kConfig.getKinesisStreams(system).forEach(stream -> { StreamConfig streamConfig = new StreamConfig(kConfig); SystemStream ss = new SystemStream(system, stream); if (streamConfig.getBootstrapEnabled(ss)) { throw new ConfigException("Kinesis streams cannot be configured as bootstrap streams."); } }); } }
streamConfigs.put(String.format(StreamConfig.PHYSICAL_NAME_FOR_STREAM_ID(), streamId), streamName); if (tableDescriptor != null) { streamConfigs.put(String.format(StreamConfig.BOOTSTRAP_FOR_STREAM_ID(), streamId), "true"); streamConfigs.put(String.format(StreamConfig.CONSUMER_OFFSET_DEFAULT_FOR_STREAM_ID(), streamId), "oldest");
@Test(expected = IllegalArgumentException.class) public void testGetStreamWithOutSystemInConfig() { Config config = buildStreamConfig(STREAM_ID, StreamConfig.PHYSICAL_NAME(), TEST_PHYSICAL_NAME); StreamSpec spec = StreamUtil.getStreamSpec(STREAM_ID, new StreamConfig(config)); assertEquals(TEST_SYSTEM, spec.getSystemName()); }
StreamEdge(StreamSpec streamSpec, boolean isIntermediate, boolean isBroadcast, Config config) { this.streamSpec = streamSpec; this.isIntermediate = isIntermediate; // broadcast can be configured either by an operator or via the configs this.isBroadcast = isBroadcast || new StreamConfig(config).getBroadcastEnabled(streamSpec.toSystemStream()); this.config = config; if (isBroadcast && isIntermediate) { partitions = 1; } this.name = StreamUtil.getNameFromSystemStream(getSystemStream()); }