public IntermediateMessageSerde(Serde userMessageSerde) { this.userMessageSerde = userMessageSerde; this.watermarkSerde = new JsonSerdeV2<>(WatermarkMessage.class); this.eosSerde = new JsonSerdeV2<>(EndOfStreamMessage.class); }
@SuppressWarnings("unchecked") @Override public String toString() { JsonSerdeV2<KinesisSystemConsumerOffset> jsonSerde = new JsonSerdeV2<>(KinesisSystemConsumerOffset.class); return new String(jsonSerde.toBytes(this)); }
static KinesisSystemConsumerOffset parse(String metadata) { JsonSerdeV2<KinesisSystemConsumerOffset> jsonSerde = new JsonSerdeV2<>(KinesisSystemConsumerOffset.class); byte[] bytes; try { bytes = metadata.getBytes("UTF-8"); } catch (Exception e) { throw new SamzaException(e); } return jsonSerde.fromBytes(bytes); }
private String serializeSystemStreamPartition(SystemStreamPartition ssp) { // Create the Json string for SystemStreamPartition Map<String, String> sspMap = new HashMap<>(); sspMap.put(SYSTEM_PROP_NAME, ssp.getSystem()); sspMap.put(STREAM_PROP_NAME, ssp.getStream()); sspMap.put(PARTITION_PROP_NAME, String.valueOf(ssp.getPartition().getPartitionId())); return new String(jsonSerde.toBytes(sspMap)); }
private SystemStreamPartition deserializeSystemStreamPartition(String serializedSSP) { Map<String, String> sspPropertiesMap = jsonSerde.fromBytes(serializedSSP.getBytes()); String systemName = sspPropertiesMap.get(SYSTEM_PROP_NAME); String streamName = sspPropertiesMap.get(STREAM_PROP_NAME); Partition partition = new Partition(Integer.parseInt(sspPropertiesMap.get("partition"))); return new SystemStreamPartition(systemName, streamName, partition); }
public JsonSerdeV2<Object> getSerde(String name, Config config) { return new JsonSerdeV2<>(); } }
private static String toStoreKey(SystemStreamPartition ssp, TaskName taskName) { return new String(new JsonSerdeV2<>().toBytes(new StartpointKey(ssp, taskName))); } }
public IntermediateMessageSerde(Serde userMessageSerde) { this.userMessageSerde = userMessageSerde; this.watermarkSerde = new JsonSerdeV2<>(WatermarkMessage.class); this.eosSerde = new JsonSerdeV2<>(EndOfStreamMessage.class); }
Assert.assertEquals(new String(new JsonSerdeV2<>().toBytes(startpointKey1)), new String(new JsonSerdeV2<>().toBytes(startpointKey2))); Assert.assertEquals(startpointKeyWithTask1, startpointKeyWithTask2); Assert.assertEquals(new String(new JsonSerdeV2<>().toBytes(startpointKeyWithTask1)), new String(new JsonSerdeV2<>().toBytes(startpointKeyWithTask2))); Assert.assertNotEquals(new String(new JsonSerdeV2<>().toBytes(startpointKey1)), new String(new JsonSerdeV2<>().toBytes(startpointKeyWithTask1))); Assert.assertNotEquals(new String(new JsonSerdeV2<>().toBytes(startpointKey1)), new String(new JsonSerdeV2<>().toBytes(startpointKeyWithDifferentSSP))); Assert.assertNotEquals(startpointKeyWithTask1, startpointKeyWithDifferentTask); Assert.assertNotEquals(new String(new JsonSerdeV2<>().toBytes(startpointKeyWithTask1)), new String(new JsonSerdeV2<>().toBytes(startpointKeyWithDifferentTask))); Assert.assertNotEquals(new String(new JsonSerdeV2<>().toBytes(startpointKeyWithTask1)), new String(new JsonSerdeV2<>().toBytes(startpointKeyWithDifferentTask)));
public IntermediateMessageSerde(Serde userMessageSerde) { this.userMessageSerde = userMessageSerde; this.watermarkSerde = new JsonSerdeV2<>(WatermarkMessage.class); this.eosSerde = new JsonSerdeV2<>(EndOfStreamMessage.class); }
public IntermediateMessageSerde(Serde userMessageSerde) { this.userMessageSerde = userMessageSerde; this.watermarkSerde = new JsonSerdeV2<>(WatermarkMessage.class); this.eosSerde = new JsonSerdeV2<>(EndOfStreamMessage.class); }
public IntermediateMessageSerde(Serde userMessageSerde) { this.userMessageSerde = userMessageSerde; this.watermarkSerde = new JsonSerdeV2<>(WatermarkMessage.class); this.eosSerde = new JsonSerdeV2<>(EndOfStreamMessage.class); }
public JsonSerdeV2<Object> getSerde(String name, Config config) { return new JsonSerdeV2<>(); } }
public static <T> JsonSerdeV2<T> of(Class<T> clazz) { return new JsonSerdeV2<>(clazz); }
public static <T> JsonSerdeV2<T> of(Class<T> clazz) { return new JsonSerdeV2<>(clazz); }
if (isTable) { tableDescriptor = new RocksDbTableDescriptor("InputTable-" + name, KVSerde.of( new JsonSerdeV2<>(SamzaSqlCompositeKey.class), new JsonSerdeV2<>(SamzaSqlRelMessage.class)));
tableFieldNames); Serde<SamzaSqlCompositeKey> keySerde = new JsonSerdeV2<>(SamzaSqlCompositeKey.class); SamzaSqlRelMessageSerdeFactory.SamzaSqlRelMessageSerde valueSerde = (SamzaSqlRelMessageSerdeFactory.SamzaSqlRelMessageSerde) new SamzaSqlRelMessageSerdeFactory().getSerde(null, null);
StreamApplication getRepartitionJoinStreamApplication() { return appDesc -> { MessageStream<KV<String, Object>> input1 = appDesc.getInputStream(input1Descriptor); MessageStream<KV<String, Object>> input2 = appDesc.getInputStream(input2Descriptor); OutputStream<KV<String, Object>> output = appDesc.getOutputStream(outputDescriptor); JoinFunction<String, Object, Object, KV<String, Object>> mockJoinFn = mock(JoinFunction.class); input1 .partitionBy(KV::getKey, KV::getValue, defaultSerde, "p1") .map(kv -> kv.value) .join(input2.map(kv -> kv.value), mockJoinFn, new StringSerde(), new JsonSerdeV2<>(Object.class), new JsonSerdeV2<>(Object.class), Duration.ofHours(1), "j1") .sendTo(output); }; }
defaultSerde = KVSerde.of(new StringSerde(), new JsonSerdeV2<>()); inputSystemDescriptor = new GenericSystemDescriptor("input-system", "mockSystemFactoryClassName"); outputSystemDescriptor = new GenericSystemDescriptor("output-system", "mockSystemFactoryClassName");
@Before public void setUp() { defaultSerde = KVSerde.of(new StringSerde(), new JsonSerdeV2<>()); inputSystemDescriptor = new GenericSystemDescriptor("input-system", "mockSystemFactoryClassName"); outputSystemDescriptor = new GenericSystemDescriptor("output-system", "mockSystemFactoryClassName"); intermediateSystemDescriptor = new GenericSystemDescriptor("intermediate-system", "mockSystemFactoryClassName"); input1Descriptor = inputSystemDescriptor.getInputDescriptor("input1", defaultSerde); input2Descriptor = inputSystemDescriptor.getInputDescriptor("input2", defaultSerde); outputDescriptor = outputSystemDescriptor.getOutputDescriptor("output", defaultSerde); intermediateInputDescriptor = intermediateSystemDescriptor.getInputDescriptor("jobName-jobId-partition_by-p1", defaultSerde) .withPhysicalName("jobName-jobId-partition_by-p1"); intermediateOutputDescriptor = intermediateSystemDescriptor.getOutputDescriptor("jobName-jobId-partition_by-p1", defaultSerde) .withPhysicalName("jobName-jobId-partition_by-p1"); broadcastInputDesriptor = intermediateSystemDescriptor.getInputDescriptor("jobName-jobId-broadcast-b1", defaultSerde) .withPhysicalName("jobName-jobId-broadcast-b1"); Map<String, String> configs = new HashMap<>(); configs.put(JobConfig.JOB_NAME(), "jobName"); configs.put(JobConfig.JOB_ID(), "jobId"); configs.putAll(input1Descriptor.toConfig()); configs.putAll(input2Descriptor.toConfig()); configs.putAll(outputDescriptor.toConfig()); configs.putAll(inputSystemDescriptor.toConfig()); configs.putAll(outputSystemDescriptor.toConfig()); configs.putAll(intermediateSystemDescriptor.toConfig()); configs.put(JobConfig.JOB_DEFAULT_SYSTEM(), intermediateSystemDescriptor.getSystemName()); mockConfig = spy(new MapConfig(configs)); mockStreamAppDesc = new StreamApplicationDescriptorImpl(getRepartitionJoinStreamApplication(), mockConfig); }