public Serde<String> getSerde(String name, Config config) { return new StringSerde(config.get("encoding", "UTF-8")); } }
public Serde<String> getSerde(String name, Config config) { return new StringSerde(config.get("encoding", "UTF-8")); } }
private void addValueSerde(Map<String, String> map) { String serdeId = "value-serde"; map.put(String.format(SerializerConfig.SERDE_SERIALIZED_INSTANCE(), serdeId), serializeSerde(new StringSerde("UTF-8"))); map.put(String.format(JavaTableConfig.STORE_MSG_SERDE, TABLE_ID), serdeId); }
private LocalTableDescriptor createTableDescriptor() { return new MockLocalTableDescriptor(TABLE_ID, new KVSerde(new StringSerde(), new IntegerSerde())); }
@Test public void testWithoutEntityConnectionConfigs() { String systemName = "eventHub"; String streamId = "input-stream"; EventHubsSystemDescriptor systemDescriptor = new EventHubsSystemDescriptor(systemName); EventHubsInputDescriptor<KV<String, String>> inputDescriptor = systemDescriptor .getInputDescriptor(streamId, "entity-namespace", "entity3", new StringSerde()); Map<String, String> generatedConfigs = inputDescriptor.toConfig(); assertEquals("eventHub", generatedConfigs.get("streams.input-stream.samza.system")); assertEquals("entity-namespace", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamId))); assertEquals("entity3", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamId))); assertNull(generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamId))); assertNull(generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamId))); assertNull(generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_CONSUMER_GROUP, streamId))); assertEquals(3, generatedConfigs.size()); // verify that there are no other configs }
@Test public void testWithoutEntityConnectionConfigs() { String systemName = "eventHub"; String streamId = "output-stream"; EventHubsSystemDescriptor systemDescriptor = new EventHubsSystemDescriptor(systemName); EventHubsOutputDescriptor<KV<String, String>> outputDescriptor = systemDescriptor .getOutputDescriptor(streamId, "entity-namespace", "entity3", new StringSerde()); Map<String, String> generatedConfigs = outputDescriptor.toConfig(); assertEquals("eventHub", generatedConfigs.get("streams.output-stream.samza.system")); assertEquals("entity-namespace", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamId))); assertEquals("entity3", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamId))); assertNull(generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamId))); assertNull(generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamId))); assertNull(generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_CONSUMER_GROUP, streamId))); assertEquals(3, generatedConfigs.size()); // verify that there are no other configs }
@Test public void testStringTimeSeriesKey() { TimeSeriesKey<String> storeKey = new TimeSeriesKey<>("test", 1, 23); TimeSeriesKeySerde<String> serde = new TimeSeriesKeySerde<>(new StringSerde("UTF-8")); byte[] serializedBytes = serde.toBytes(storeKey); TimeSeriesKey<String> deserializedTimeSeriesKey = serde.fromBytes(serializedBytes); assertEquals(storeKey.getKey(), deserializedTimeSeriesKey.getKey()); assertEquals(storeKey.getSeqNum(), deserializedTimeSeriesKey.getSeqNum()); assertEquals(storeKey.getTimestamp(), deserializedTimeSeriesKey.getTimestamp()); assertEquals(storeKey, deserializedTimeSeriesKey); }
@Test public void testNullTimeSeriesKey() { TimeSeriesKey<String> storeKey = new TimeSeriesKey<>(null, 1, 23); TimeSeriesKeySerde<String> serde = new TimeSeriesKeySerde<>(new StringSerde("UTF-8")); byte[] serializedBytes = serde.toBytes(storeKey); TimeSeriesKey<String> deserializedTimeSeriesKey = serde.fromBytes(serializedBytes); assertEquals(storeKey.getKey(), deserializedTimeSeriesKey.getKey()); assertEquals(storeKey.getSeqNum(), deserializedTimeSeriesKey.getSeqNum()); assertEquals(storeKey.getTimestamp(), deserializedTimeSeriesKey.getTimestamp()); assertEquals(storeKey, deserializedTimeSeriesKey); }
@Test public void testEntityConnectionConfigs() { String systemName = "eventHub"; String streamId = "output-stream"; EventHubsSystemDescriptor systemDescriptor = new EventHubsSystemDescriptor(systemName); EventHubsOutputDescriptor<KV<String, String>> outputDescriptor = systemDescriptor .getOutputDescriptor(streamId, "entity-namespace", "entity3", new StringSerde()) .withSasKeyName("secretkey") .withSasKey("sasToken-123"); Map<String, String> generatedConfigs = outputDescriptor.toConfig(); assertEquals("eventHub", generatedConfigs.get("streams.output-stream.samza.system")); assertEquals("entity-namespace", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_NAMESPACE, streamId))); assertEquals("entity3", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_ENTITYPATH, streamId))); assertEquals("secretkey", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_SAS_KEY_NAME, streamId))); assertEquals("sasToken-123", generatedConfigs.get(String.format(EventHubConfig.CONFIG_STREAM_SAS_TOKEN, streamId))); }
@Test public void testGetWithNonExistentKeys() { TimeSeriesStore<String, byte[]> timeSeriesStore = newTimeSeriesStore(new StringSerde("UTF-8"), true); timeSeriesStore.put("hello", "world-1".getBytes(), 1L); // read from a non-existent key List<TimestampedValue<byte[]>> values = readStore(timeSeriesStore, "non-existent-key", 0, Integer.MAX_VALUE); Assert.assertEquals(0, values.size()); // read from an existing key but out of range timestamp values = readStore(timeSeriesStore, "hello", 2, Integer.MAX_VALUE); Assert.assertEquals(0, values.size()); }
@Test public void testDeletesInOverwriteMode() { // instantiate a store in overwrite mode TimeSeriesStore<String, byte[]> timeSeriesStore = newTimeSeriesStore(new StringSerde("UTF-8"), false); // insert an entry with key "hello" at timestamps "1" and "2" timeSeriesStore.put("hello", "world-1".getBytes(), 1L); timeSeriesStore.put("hello", "world-1".getBytes(), 2L); timeSeriesStore.put("hello", "world-2".getBytes(), 2L); List<TimestampedValue<byte[]>> values = readStore(timeSeriesStore, "hello", 1L, 3L); Assert.assertEquals(2, values.size()); timeSeriesStore.remove("hello", 0L, 3L); values = readStore(timeSeriesStore, "hello", 1L, 3L); Assert.assertEquals(0, values.size()); }
@Test public void testISDConfigsWithDefaults() { KafkaSystemDescriptor sd = new KafkaSystemDescriptor("kafka") .withConsumerZkConnect(ImmutableList.of("localhost:123")) .withProducerBootstrapServers(ImmutableList.of("localhost:567", "localhost:890")); KafkaInputDescriptor<KV<String, Integer>> isd = sd.getInputDescriptor("input-stream", KVSerde.of(new StringSerde(), new IntegerSerde())); Map<String, String> generatedConfigs = isd.toConfig(); assertEquals("kafka", generatedConfigs.get("streams.input-stream.samza.system")); assertEquals(1, generatedConfigs.size()); // verify that there are no other configs } }
@Test public void testMissingInputDescriptorFields() { String systemName = "eventHub"; String streamId = "input-stream"; EventHubsSystemDescriptor systemDescriptor = new EventHubsSystemDescriptor(systemName); try { systemDescriptor.getInputDescriptor(streamId, null, null, new StringSerde()); fail("Should have thrown Config Exception"); } catch (ConfigException exception) { assertEquals(String.format("Missing namespace and entity path Event Hubs input descriptor in " // + "system: {%s}, stream: {%s}", systemName, streamId), exception.getMessage()); } }
@Test public void testMissingOutputDescriptorFields() { String systemName = "eventHub"; String streamId = "input-stream"; EventHubsSystemDescriptor systemDescriptor = new EventHubsSystemDescriptor(systemName); try { systemDescriptor.getOutputDescriptor(streamId, null, null, new StringSerde()); fail("Should have thrown Config Exception"); } catch (ConfigException exception) { assertEquals(String.format("Missing namespace and entity path Event Hubs output descriptor in " // + "system: {%s}, stream: {%s}", systemName, streamId), exception.getMessage()); } }
@Test public void testISDConfigsWithOverrides() { KafkaSystemDescriptor sd = new KafkaSystemDescriptor("kafka"); KafkaInputDescriptor<KV<String, Integer>> isd = sd.getInputDescriptor("input-stream", KVSerde.of(new StringSerde(), new IntegerSerde())) .withConsumerAutoOffsetReset("largest") .withConsumerFetchMessageMaxBytes(1024 * 1024); Map<String, String> generatedConfigs = isd.toConfig();; assertEquals("kafka", generatedConfigs.get("streams.input-stream.samza.system")); assertEquals("largest", generatedConfigs.get("systems.kafka.streams.input-stream.consumer.auto.offset.reset")); assertEquals("1048576", generatedConfigs.get("systems.kafka.streams.input-stream.consumer.fetch.message.max.bytes")); }
@Test public void testWithSerdes() { List<TableDescriptor> descriptors = Arrays.asList( new MockLocalTableDescriptor("t1", KVSerde.of(new StringSerde(), new IntegerSerde())), new MockLocalTableDescriptor("t2", KVSerde.of(new StringSerde(), new IntegerSerde())) ); Config jobConfig = new MapConfig(TableConfigGenerator.generateSerdeConfig(descriptors)); JavaTableConfig javaTableConfig = new JavaTableConfig(jobConfig); assertNotNull(javaTableConfig.getKeySerde("t1")); assertNotNull(javaTableConfig.getMsgSerde("t1")); assertNotNull(javaTableConfig.getKeySerde("t2")); assertNotNull(javaTableConfig.getMsgSerde("t2")); MapConfig tableConfig = new MapConfig(TableConfigGenerator.generate(jobConfig, descriptors)); javaTableConfig = new JavaTableConfig(tableConfig); assertNotNull(javaTableConfig.getTableProviderFactory("t1")); assertNotNull(javaTableConfig.getTableProviderFactory("t2")); }
StreamApplication getRepartitionJoinStreamApplication() { return appDesc -> { MessageStream<KV<String, Object>> input1 = appDesc.getInputStream(input1Descriptor); MessageStream<KV<String, Object>> input2 = appDesc.getInputStream(input2Descriptor); OutputStream<KV<String, Object>> output = appDesc.getOutputStream(outputDescriptor); JoinFunction<String, Object, Object, KV<String, Object>> mockJoinFn = mock(JoinFunction.class); input1 .partitionBy(KV::getKey, KV::getValue, defaultSerde, "p1") .map(kv -> kv.value) .join(input2.map(kv -> kv.value), mockJoinFn, new StringSerde(), new JsonSerdeV2<>(Object.class), new JsonSerdeV2<>(Object.class), Duration.ofHours(1), "j1") .sendTo(output); }; }
@Test public void testStreamDescriptorContainsKVserde() { String systemName = "eventHub"; String streamId = "output-stream"; EventHubsSystemDescriptor systemDescriptor = new EventHubsSystemDescriptor(systemName); EventHubsOutputDescriptor<KV<String, String>> outputDescriptor = systemDescriptor .getOutputDescriptor(streamId, "entity-namespace", "entity3", new StringSerde()); assertTrue(outputDescriptor.getSerde() instanceof KVSerde); assertTrue(((KVSerde) outputDescriptor.getSerde()).getKeySerde() instanceof NoOpSerde); assertTrue(((KVSerde) outputDescriptor.getSerde()).getValueSerde() instanceof StringSerde); } }
@Test public void testStreamDescriptorContainsKVserde() { String systemName = "eventHub"; String streamId = "input-stream"; EventHubsSystemDescriptor systemDescriptor = new EventHubsSystemDescriptor(systemName); EventHubsInputDescriptor<KV<String, String>> outputDescriptor = systemDescriptor .getInputDescriptor(streamId, "entity-namespace", "entity3", new StringSerde()); assertTrue(outputDescriptor.getSerde() instanceof KVSerde); assertTrue(((KVSerde) outputDescriptor.getSerde()).getKeySerde() instanceof NoOpSerde); assertTrue(((KVSerde) outputDescriptor.getSerde()).getValueSerde() instanceof StringSerde); } }
@Test public void testBroadcastOperatorSpec() { OutputStreamImpl<TestOutputMessageEnvelope> outputStream = new OutputStreamImpl<>("output-0", new StringSerde("UTF-8"), new JsonSerdeV2<TestOutputMessageEnvelope>(), true); BroadcastOperatorSpec<TestOutputMessageEnvelope> broadcastOpSpec = new BroadcastOperatorSpec<>(outputStream, "broadcast-1"); BroadcastOperatorSpec<TestOutputMessageEnvelope> broadcastOpCopy = (BroadcastOperatorSpec<TestOutputMessageEnvelope>) OperatorSpecTestUtils .copyOpSpec(broadcastOpSpec); assertNotEquals(broadcastOpCopy, broadcastOpSpec); assertEquals(broadcastOpCopy.getOpId(), broadcastOpSpec.getOpId()); assertTrue(broadcastOpCopy.getOutputStream() != broadcastOpSpec.getOutputStream()); assertEquals(broadcastOpCopy.getOutputStream().getStreamId(), broadcastOpSpec.getOutputStream().getStreamId()); assertEquals(broadcastOpCopy.getOutputStream().isKeyed(), broadcastOpSpec.getOutputStream().isKeyed()); }