public Serde<Integer> getSerde(String name, Config config) { return new IntegerSerde(); } }
public Serde<Integer> getSerde(String name, Config config) { return new IntegerSerde(); } }
private void addKeySerde(Map<String, String> map) { String serdeId = "key-serde"; map.put(String.format(SerializerConfig.SERDE_SERIALIZED_INSTANCE(), serdeId), serializeSerde(new IntegerSerde())); map.put(String.format(JavaTableConfig.STORE_KEY_SERDE, TABLE_ID), serdeId); }
@Test public void testEmptyValueSerialization() { byte[] expectedBytes = new byte[8]; ByteBuffer.wrap(expectedBytes).putLong(1234L); TimestampedValueSerde<Integer> timestampedValueSerde = new TimestampedValueSerde<>(new IntegerSerde()); TimestampedValue<Integer> timestampedValue = new TimestampedValue<>(null, 1234L); assertTrue(Arrays.equals(expectedBytes, timestampedValueSerde.toBytes(timestampedValue))); } }
private LocalTableDescriptor createTableDescriptor() { return new MockLocalTableDescriptor(TABLE_ID, new KVSerde(new StringSerde(), new IntegerSerde())); }
public void testAPIUsage() { // does not assert anything, but acts as a compile-time check on expected descriptor type parameters // and validates that the method calls can be chained. ExampleExpandingSystemDescriptor expandingSystem = new ExampleExpandingSystemDescriptor("expandingSystem"); ExampleExpandingInputDescriptor<Long> input1 = expandingSystem.getInputDescriptor("input1", new IntegerSerde()); ExampleExpandingOutputDescriptor<Integer> output1 = expandingSystem.getOutputDescriptor("output1", new IntegerSerde()); input1 .shouldBootstrap() .withOffsetDefault(SystemStreamMetadata.OffsetType.NEWEST) .withPriority(1) .shouldResetOffset() .withStreamConfigs(Collections.emptyMap()); output1 .withStreamConfigs(Collections.emptyMap()); }
@Test public void testISDConfigsWithDefaults() { KafkaSystemDescriptor sd = new KafkaSystemDescriptor("kafka") .withConsumerZkConnect(ImmutableList.of("localhost:123")) .withProducerBootstrapServers(ImmutableList.of("localhost:567", "localhost:890")); KafkaInputDescriptor<KV<String, Integer>> isd = sd.getInputDescriptor("input-stream", KVSerde.of(new StringSerde(), new IntegerSerde())); Map<String, String> generatedConfigs = isd.toConfig(); assertEquals("kafka", generatedConfigs.get("streams.input-stream.samza.system")); assertEquals(1, generatedConfigs.size()); // verify that there are no other configs } }
@Test public void testISDObjectsWithOverrides() { GenericSystemDescriptor mySystem = new GenericSystemDescriptor("input-system", "factory.class.name") .withSystemConfigs(Collections.emptyMap()) .withDefaultStreamConfigs(Collections.emptyMap()); IntegerSerde streamSerde = new IntegerSerde(); GenericInputDescriptor<Integer> isd = mySystem.getInputDescriptor("input-stream", streamSerde); assertEquals(streamSerde, isd.getSerde()); assertFalse(isd.getTransformer().isPresent()); } }
@Test public void testAPIUsage() { // does not assert anything, but acts as a compile-time check on expected descriptor type parameters // and validates that the method calls can be chained. ExampleTransformingSystemDescriptor imeTransformingSystem = new ExampleTransformingSystemDescriptor("imeTransformingSystem") .withSystemConfigs(Collections.emptyMap()); ExampleTransformingInputDescriptor<Long> input1 = imeTransformingSystem.getInputDescriptor("input1", new IntegerSerde()); ExampleTransformingOutputDescriptor<Integer> output1 = imeTransformingSystem.getOutputDescriptor("output1", new IntegerSerde()); input1 .shouldBootstrap() .withOffsetDefault(SystemStreamMetadata.OffsetType.NEWEST) .withPriority(1) .shouldResetOffset() .withStreamConfigs(Collections.emptyMap()); output1 .withStreamConfigs(Collections.emptyMap()); }
@Test public void testISDObjectsWithOverrides() { ExampleExpandingSystemDescriptor expandingSystem = new ExampleExpandingSystemDescriptor("expandingSystem"); IntegerSerde streamSerde = new IntegerSerde(); ExampleExpandingInputDescriptor<Long> expandingISD = expandingSystem.getInputDescriptor("input-stream", streamSerde); assertEquals(streamSerde, expandingISD.getSerde()); assertEquals(expandingSystem.getTransformer().get(), expandingISD.getTransformer().get()); } }
@Test public void testISDObjectsWithOverrides() { ExampleTransformingSystemDescriptor imeTransformingSystem = new ExampleTransformingSystemDescriptor("imeTransformingSystem"); IntegerSerde streamSerde = new IntegerSerde(); ExampleTransformingInputDescriptor<Long> overridingISD = imeTransformingSystem.getInputDescriptor("input-stream", streamSerde); assertEquals(streamSerde, overridingISD.getSerde()); assertEquals(imeTransformingSystem.getTransformer().get(), overridingISD.getTransformer().get()); } }
private StreamApplicationDescriptorImpl getKeyedSessionWindowStreamGraph(AccumulationMode mode, Duration duration) throws IOException { StreamApplication userApp = appDesc -> { KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde()); GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass"); GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde); appDesc.getInputStream(inputDescriptor) .window(Windows.keyedSessionWindow(KV::getKey, duration, new IntegerSerde(), kvSerde) .setAccumulationMode(mode), "w1") .sink((message, messageCollector, taskCoordinator) -> { SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream"); messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message)); }); }; return new StreamApplicationDescriptorImpl(userApp, config); }
@Test public void testAPIUsage() { // does not assert anything, but acts as a compile-time check on expected descriptor type parameters // and validates that the method calls can be chained. ExampleSimpleSystemDescriptor kafkaSystem = new ExampleSimpleSystemDescriptor("kafka-system") .withSystemConfigs(Collections.emptyMap()); ExampleSimpleInputDescriptor<Integer> input1 = kafkaSystem.getInputDescriptor("input1", new IntegerSerde()); ExampleSimpleOutputDescriptor<Integer> output1 = kafkaSystem.getOutputDescriptor("output1", new IntegerSerde()); input1 .shouldBootstrap() .withOffsetDefault(SystemStreamMetadata.OffsetType.NEWEST) .withPriority(1) .shouldResetOffset() .withStreamConfigs(Collections.emptyMap()); output1 .withStreamConfigs(Collections.emptyMap()); }
@Test public void testISDConfigsWithOverrides() { KafkaSystemDescriptor sd = new KafkaSystemDescriptor("kafka"); KafkaInputDescriptor<KV<String, Integer>> isd = sd.getInputDescriptor("input-stream", KVSerde.of(new StringSerde(), new IntegerSerde())) .withConsumerAutoOffsetReset("largest") .withConsumerFetchMessageMaxBytes(1024 * 1024); Map<String, String> generatedConfigs = isd.toConfig();; assertEquals("kafka", generatedConfigs.get("streams.input-stream.samza.system")); assertEquals("largest", generatedConfigs.get("systems.kafka.streams.input-stream.consumer.auto.offset.reset")); assertEquals("1048576", generatedConfigs.get("systems.kafka.streams.input-stream.consumer.fetch.message.max.bytes")); }
private StreamApplicationDescriptorImpl getKeyedTumblingWindowStreamGraph(AccumulationMode mode, Duration duration, Trigger<KV<Integer, Integer>> earlyTrigger) throws IOException { StreamApplication userApp = appDesc -> { KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde()); GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass"); GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde); appDesc.getInputStream(inputDescriptor) .window(Windows.keyedTumblingWindow(KV::getKey, duration, new IntegerSerde(), kvSerde) .setEarlyTrigger(earlyTrigger).setAccumulationMode(mode), "w1") .sink((message, messageCollector, taskCoordinator) -> { SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream"); messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message)); }); }; return new StreamApplicationDescriptorImpl(userApp, config); }
@Test public void testISDObjectsWithOverrides() { ExampleSimpleSystemDescriptor ssd = new ExampleSimpleSystemDescriptor("kafka-system"); IntegerSerde streamSerde = new IntegerSerde(); ExampleSimpleInputDescriptor<Integer> isd = ssd.getInputDescriptor("input-stream", streamSerde); assertEquals(streamSerde, isd.getSerde()); assertFalse(isd.getTransformer().isPresent()); } }
@Test(expected = SamzaException.class) public void joinWithSelfThrowsException() throws Exception { Map<String, String> mapConfig = new HashMap<>(); mapConfig.put("job.name", "jobName"); mapConfig.put("job.id", "jobId"); StreamTestUtils.addStreamConfigs(mapConfig, "inStream", "insystem", "instream"); Config config = new MapConfig(mapConfig); StreamApplicationDescriptorImpl streamAppDesc = new StreamApplicationDescriptorImpl(appDesc -> { IntegerSerde integerSerde = new IntegerSerde(); KVSerde<Integer, Integer> kvSerde = KVSerde.of(integerSerde, integerSerde); GenericSystemDescriptor sd = new GenericSystemDescriptor("insystem", "mockFactoryClassName"); GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("inStream", kvSerde); MessageStream<KV<Integer, Integer>> inStream = appDesc.getInputStream(inputDescriptor); inStream.join(inStream, new TestJoinFunction(), integerSerde, kvSerde, kvSerde, JOIN_TTL, "join"); }, config); createStreamOperatorTask(new SystemClock(), streamAppDesc); // should throw an exception }
private StreamApplicationDescriptorImpl getAggregateTumblingWindowStreamGraph(AccumulationMode mode, Duration timeDuration, Trigger<IntegerEnvelope> earlyTrigger) throws IOException { StreamApplication userApp = appDesc -> { KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde()); GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass"); GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde); MessageStream<KV<Integer, Integer>> integers = appDesc.getInputStream(inputDescriptor); integers .map(new KVMapFunction()) .window(Windows.<IntegerEnvelope, Integer>tumblingWindow(timeDuration, () -> 0, (m, c) -> c + 1, new IntegerSerde()) .setEarlyTrigger(earlyTrigger) .setAccumulationMode(mode), "w1") .sink((message, messageCollector, taskCoordinator) -> { SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream"); messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message)); }); }; return new StreamApplicationDescriptorImpl(userApp, config); }
@Test public void testWithSerdes() { List<TableDescriptor> descriptors = Arrays.asList( new MockLocalTableDescriptor("t1", KVSerde.of(new StringSerde(), new IntegerSerde())), new MockLocalTableDescriptor("t2", KVSerde.of(new StringSerde(), new IntegerSerde())) ); Config jobConfig = new MapConfig(TableConfigGenerator.generateSerdeConfig(descriptors)); JavaTableConfig javaTableConfig = new JavaTableConfig(jobConfig); assertNotNull(javaTableConfig.getKeySerde("t1")); assertNotNull(javaTableConfig.getMsgSerde("t1")); assertNotNull(javaTableConfig.getKeySerde("t2")); assertNotNull(javaTableConfig.getMsgSerde("t2")); MapConfig tableConfig = new MapConfig(TableConfigGenerator.generate(jobConfig, descriptors)); javaTableConfig = new JavaTableConfig(tableConfig); assertNotNull(javaTableConfig.getTableProviderFactory("t1")); assertNotNull(javaTableConfig.getTableProviderFactory("t2")); }
private StreamApplicationDescriptorImpl getTumblingWindowStreamGraph(AccumulationMode mode, Duration duration, Trigger<KV<Integer, Integer>> earlyTrigger) throws IOException { StreamApplication userApp = appDesc -> { KVSerde<Integer, Integer> kvSerde = KVSerde.of(new IntegerSerde(), new IntegerSerde()); GenericSystemDescriptor sd = new GenericSystemDescriptor("kafka", "mockFactoryClass"); GenericInputDescriptor<KV<Integer, Integer>> inputDescriptor = sd.getInputDescriptor("integers", kvSerde); appDesc.getInputStream(inputDescriptor) .window(Windows.tumblingWindow(duration, kvSerde).setEarlyTrigger(earlyTrigger) .setAccumulationMode(mode), "w1") .sink((message, messageCollector, taskCoordinator) -> { SystemStream outputSystemStream = new SystemStream("outputSystem", "outputStream"); messageCollector.send(new OutgoingMessageEnvelope(outputSystemStream, message)); }); }; return new StreamApplicationDescriptorImpl(userApp, config); }