public static StreamSpec getStreamSpec(String streamId, StreamConfig streamConfig) { String physicalName = streamConfig.getPhysicalName(streamId); String system = streamConfig.getSystem(streamId); Map<String, String> streamProperties = streamConfig.getStreamProperties(streamId); return new StreamSpec(streamId, physicalName, system, streamProperties); } }
public static StreamSpec getStreamSpec(String streamId, StreamConfig streamConfig) { String physicalName = streamConfig.getPhysicalName(streamId); String system = streamConfig.getSystem(streamId); Map<String, String> streamProperties = streamConfig.getStreamProperties(streamId); return new StreamSpec(streamId, physicalName, system, streamProperties); } }
public static StreamSpec getStreamSpec(String streamId, StreamConfig streamConfig) { String physicalName = streamConfig.getPhysicalName(streamId); String system = streamConfig.getSystem(streamId); Map<String, String> streamProperties = streamConfig.getStreamProperties(streamId); return new StreamSpec(streamId, physicalName, system, streamProperties); } }
public static StreamSpec getStreamSpec(String streamId, StreamConfig streamConfig) { String physicalName = streamConfig.getPhysicalName(streamId); String system = streamConfig.getSystem(streamId); Map<String, String> streamProperties = streamConfig.getStreamProperties(streamId); return new StreamSpec(streamId, physicalName, system, streamProperties); } }
public static StreamSpec getStreamSpec(String streamId, StreamConfig streamConfig) { String physicalName = streamConfig.getPhysicalName(streamId); String system = streamConfig.getSystem(streamId); Map<String, String> streamProperties = streamConfig.getStreamProperties(streamId); return new StreamSpec(streamId, physicalName, system, streamProperties); } }
String inputSystem1 = streamConfig.getSystem(inputStreamId1); String inputPhysicalName1 = streamConfig.getPhysicalName(inputStreamId1); String inputStreamId2 = specGraph.getInputOperators().keySet().stream().skip(1).findFirst().get(); String inputSystem2 = streamConfig.getSystem(inputStreamId2); String inputPhysicalName2 = streamConfig.getPhysicalName(inputStreamId2); String outputSystem = streamConfig.getSystem(outputStreamId); String outputPhysicalName = streamConfig.getPhysicalName(outputStreamId);
String inputSystem = streamConfig.getSystem(inputStreamId); String inputPhysicalName = streamConfig.getPhysicalName(inputStreamId); String outputStreamId1 = specGraph.getOutputStreams().keySet().stream().findFirst().get(); String outputSystem1 = streamConfig.getSystem(outputStreamId1); String outputPhysicalName1 = streamConfig.getPhysicalName(outputStreamId1); String outputStreamId2 = specGraph.getOutputStreams().keySet().stream().skip(1).findFirst().get(); String outputSystem2 = streamConfig.getSystem(outputStreamId2); String outputPhysicalName2 = streamConfig.getPhysicalName(outputStreamId2);
@Test public void testTranslateComplex() { Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(10); config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, "Insert into testavro.outputTopic(string_value) select Flatten(array_values) from testavro.COMPLEX1"); Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config)); List<String> sqlStmts = fetchSqlFromConfig(config); List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts); SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream) .collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList())); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(streamApp -> { }, samzaConfig); QueryTranslator translator = new QueryTranslator(streamAppDesc, samzaSqlApplicationConfig); translator.translate(queryInfo.get(0), streamAppDesc, 0); OperatorSpecGraph specGraph = streamAppDesc.getOperatorSpecGraph(); StreamConfig streamConfig = new StreamConfig(samzaConfig); String inputStreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String inputSystem = streamConfig.getSystem(inputStreamId); String inputPhysicalName = streamConfig.getPhysicalName(inputStreamId); String outputStreamId = specGraph.getOutputStreams().keySet().stream().findFirst().get(); String outputSystem = streamConfig.getSystem(outputStreamId); String outputPhysicalName = streamConfig.getPhysicalName(outputStreamId); Assert.assertEquals(1, specGraph.getOutputStreams().size()); Assert.assertEquals("testavro", outputSystem); Assert.assertEquals("outputTopic", outputPhysicalName); Assert.assertEquals(1, specGraph.getInputOperators().size()); Assert.assertEquals("testavro", inputSystem); Assert.assertEquals("COMPLEX1", inputPhysicalName); }
String inputSystem1 = streamConfig.getSystem(inputStreamId1); String inputPhysicalName1 = streamConfig.getPhysicalName(inputStreamId1); String inputStreamId2 = specGraph.getInputOperators().keySet().stream().skip(1).findFirst().get(); String inputSystem2 = streamConfig.getSystem(inputStreamId2); String inputPhysicalName2 = streamConfig.getPhysicalName(inputStreamId2); String outputSystem1 = streamConfig.getSystem(outputStreamId1); String outputPhysicalName1 = streamConfig.getPhysicalName(outputStreamId1); String outputStreamId2 = specGraph.getOutputStreams().keySet().stream().skip(1).findFirst().get(); String outputSystem2 = streamConfig.getSystem(outputStreamId2); String outputPhysicalName2 = streamConfig.getPhysicalName(outputStreamId2);
String input1System = streamConfig.getSystem(input1StreamId); String input1PhysicalName = streamConfig.getPhysicalName(input1StreamId); String input2StreamId = specGraph.getInputOperators().keySet().stream().skip(1).findFirst().get(); String input2System = streamConfig.getSystem(input2StreamId); String input2PhysicalName = streamConfig.getPhysicalName(input2StreamId); String input3StreamId = specGraph.getInputOperators().keySet().stream().skip(2).findFirst().get(); String input3System = streamConfig.getSystem(input3StreamId); String input3PhysicalName = streamConfig.getPhysicalName(input3StreamId); String input4StreamId = specGraph.getInputOperators().keySet().stream().skip(3).findFirst().get(); String input4System = streamConfig.getSystem(input4StreamId); String input4PhysicalName = streamConfig.getPhysicalName(input4StreamId); String output1StreamId = specGraph.getOutputStreams().keySet().stream().findFirst().get(); String output1System = streamConfig.getSystem(output1StreamId); String output1PhysicalName = streamConfig.getPhysicalName(output1StreamId); String output2StreamId = specGraph.getOutputStreams().keySet().stream().skip(1).findFirst().get(); String output2System = streamConfig.getSystem(output2StreamId); String output2PhysicalName = streamConfig.getPhysicalName(output2StreamId); String output3StreamId = specGraph.getOutputStreams().keySet().stream().skip(2).findFirst().get(); String output3System = streamConfig.getSystem(output3StreamId); String output3PhysicalName = streamConfig.getPhysicalName(output3StreamId);
String input1System = streamConfig.getSystem(input1StreamId); String input1PhysicalName = streamConfig.getPhysicalName(input1StreamId); String input2StreamId = specGraph.getInputOperators().keySet().stream().skip(1).findFirst().get(); String input2System = streamConfig.getSystem(input2StreamId); String input2PhysicalName = streamConfig.getPhysicalName(input2StreamId); String input3StreamId = specGraph.getInputOperators().keySet().stream().skip(2).findFirst().get(); String input3System = streamConfig.getSystem(input3StreamId); String input3PhysicalName = streamConfig.getPhysicalName(input3StreamId); String input4StreamId = specGraph.getInputOperators().keySet().stream().skip(3).findFirst().get(); String input4System = streamConfig.getSystem(input4StreamId); String input4PhysicalName = streamConfig.getPhysicalName(input4StreamId); String output1StreamId = specGraph.getOutputStreams().keySet().stream().findFirst().get(); String output1System = streamConfig.getSystem(output1StreamId); String output1PhysicalName = streamConfig.getPhysicalName(output1StreamId); String output2StreamId = specGraph.getOutputStreams().keySet().stream().skip(1).findFirst().get(); String output2System = streamConfig.getSystem(output2StreamId); String output2PhysicalName = streamConfig.getPhysicalName(output2StreamId); String output3StreamId = specGraph.getOutputStreams().keySet().stream().skip(2).findFirst().get(); String output3System = streamConfig.getSystem(output3StreamId); String output3PhysicalName = streamConfig.getPhysicalName(output3StreamId);
@Test public void testTranslate() { Map<String, String> config = SamzaSqlTestConfig.fetchStaticConfigsWithFactories(10); config.put(SamzaSqlApplicationConfig.CFG_SQL_STMT, "Insert into testavro.outputTopic(id) select MyTest(id) from testavro.level1.level2.SIMPLE1 as s where s.id = 10"); Config samzaConfig = SamzaSqlApplicationRunner.computeSamzaConfigs(true, new MapConfig(config)); List<String> sqlStmts = fetchSqlFromConfig(config); List<SamzaSqlQueryParser.QueryInfo> queryInfo = fetchQueryInfo(sqlStmts); SamzaSqlApplicationConfig samzaSqlApplicationConfig = new SamzaSqlApplicationConfig(new MapConfig(config), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSources).flatMap(Collection::stream) .collect(Collectors.toList()), queryInfo.stream().map(SamzaSqlQueryParser.QueryInfo::getSink).collect(Collectors.toList())); StreamApplicationDescriptorImpl appDesc = new StreamApplicationDescriptorImpl(streamApp -> { },samzaConfig); QueryTranslator translator = new QueryTranslator(appDesc, samzaSqlApplicationConfig); translator.translate(queryInfo.get(0), appDesc, 0); OperatorSpecGraph specGraph = appDesc.getOperatorSpecGraph(); StreamConfig streamConfig = new StreamConfig(samzaConfig); String inputStreamId = specGraph.getInputOperators().keySet().stream().findFirst().get(); String inputSystem = streamConfig.getSystem(inputStreamId); String inputPhysicalName = streamConfig.getPhysicalName(inputStreamId); String outputStreamId = specGraph.getOutputStreams().keySet().stream().findFirst().get(); String outputSystem = streamConfig.getSystem(outputStreamId); String outputPhysicalName = streamConfig.getPhysicalName(outputStreamId); Assert.assertEquals(1, specGraph.getOutputStreams().size()); Assert.assertEquals("testavro", outputSystem); Assert.assertEquals("outputTopic", outputPhysicalName); Assert.assertEquals(1, specGraph.getInputOperators().size()); Assert.assertEquals("testavro", inputSystem); Assert.assertEquals("SIMPLE1", inputPhysicalName); }
String input1System = streamConfig.getSystem(input1StreamId); String input1PhysicalName = streamConfig.getPhysicalName(input1StreamId); String input2StreamId = specGraph.getInputOperators().keySet().stream().skip(1).findFirst().get(); String input2System = streamConfig.getSystem(input2StreamId); String input2PhysicalName = streamConfig.getPhysicalName(input2StreamId); String input3StreamId = specGraph.getInputOperators().keySet().stream().skip(2).findFirst().get(); String input3System = streamConfig.getSystem(input3StreamId); String input3PhysicalName = streamConfig.getPhysicalName(input3StreamId); String input4StreamId = specGraph.getInputOperators().keySet().stream().skip(3).findFirst().get(); String input4System = streamConfig.getSystem(input4StreamId); String input4PhysicalName = streamConfig.getPhysicalName(input4StreamId); String output1StreamId = specGraph.getOutputStreams().keySet().stream().findFirst().get(); String output1System = streamConfig.getSystem(output1StreamId); String output1PhysicalName = streamConfig.getPhysicalName(output1StreamId); String output2StreamId = specGraph.getOutputStreams().keySet().stream().skip(1).findFirst().get(); String output2System = streamConfig.getSystem(output2StreamId); String output2PhysicalName = streamConfig.getPhysicalName(output2StreamId); String output3StreamId = specGraph.getOutputStreams().keySet().stream().skip(2).findFirst().get(); String output3System = streamConfig.getSystem(output3StreamId); String output3PhysicalName = streamConfig.getPhysicalName(output3StreamId);
String inputSystem = streamConfig.getSystem(inputStreamId); String inputPhysicalName = streamConfig.getPhysicalName(inputStreamId); String outputStreamId = specGraph.getOutputStreams().keySet().stream().findFirst().get(); String outputSystem = streamConfig.getSystem(outputStreamId); String outputPhysicalName = streamConfig.getPhysicalName(outputStreamId);
.map(id -> new StreamSpec(id, streamConfig.getPhysicalName(id), streamConfig.getSystem(id))) .collect(Collectors.toSet()); intStreams.forEach(stream -> {
.map(id -> new StreamSpec(id, streamConfig.getPhysicalName(id), streamConfig.getSystem(id))) .collect(Collectors.toSet()); intStreams.forEach(stream -> {
.map(id -> new StreamSpec(id, streamConfig.getPhysicalName(id), streamConfig.getSystem(id))) .collect(Collectors.toSet()); intStreams.forEach(stream -> {
.map(id -> new StreamSpec(id, streamConfig.getPhysicalName(id), streamConfig.getSystem(id))) .collect(Collectors.toSet()); intStreams.forEach(stream -> {
.map(id -> new StreamSpec(id, streamConfig.getPhysicalName(id), streamConfig.getSystem(id))) .collect(Collectors.toSet()); intStreams.forEach(stream -> {
@Test public void testGenerateConfig() { // an example unbounded IO stream StreamSpec spec = new StreamSpec("stream-1", "physical-stream-1", "system-1", Collections.singletonMap("property1", "haha")); StreamEdge edge = new StreamEdge(spec, false, false, new MapConfig()); Config config = edge.generateConfig(); StreamConfig streamConfig = new StreamConfig(config); assertEquals(streamConfig.getSystem(spec.getId()), "system-1"); assertEquals(streamConfig.getPhysicalName(spec.getId()), "physical-stream-1"); assertEquals(streamConfig.getIsIntermediateStream(spec.getId()), false); assertEquals(streamConfig.getStreamProperties(spec.getId()).get("property1"), "haha"); // bounded stream spec = new StreamSpec("stream-1", "physical-stream-1", "system-1", Collections.singletonMap("property1", "haha")); edge = new StreamEdge(spec, false, false, new MapConfig()); config = edge.generateConfig(); streamConfig = new StreamConfig(config); // intermediate stream edge = new StreamEdge(spec, true, false, new MapConfig()); config = edge.generateConfig(); streamConfig = new StreamConfig(config); assertEquals(streamConfig.getIsIntermediateStream(spec.getId()), true); assertEquals(streamConfig.getDefaultStreamOffset(spec.toSystemStream()).get(), "oldest"); assertEquals(streamConfig.getPriority(spec.toSystemStream()), Integer.MAX_VALUE); } }