/** * Constructs an {@link InputDescriptor} instance. * * @param streamId id of the stream * @param valueSerde serde the values in the messages in the stream * @param systemDescriptor system descriptor this stream descriptor was obtained from */ <T> KinesisInputDescriptor(String streamId, Serde<T> valueSerde, SystemDescriptor systemDescriptor) { super(streamId, KVSerde.of(new NoOpSerde<>(), valueSerde), systemDescriptor, null); }
KV<Serde, Serde> getOrCreateTableSerdes(String tableId, KVSerde kvSerde) { Serde keySerde, valueSerde; keySerde = kvSerde.getKeySerde(); valueSerde = kvSerde.getValueSerde(); if (!tableSerdes.containsKey(tableId)) { tableSerdes.put(tableId, KV.of(keySerde, valueSerde)); return tableSerdes.get(tableId); } KV<Serde, Serde> currentSerdePair = tableSerdes.get(tableId); if (!currentSerdePair.getKey().equals(keySerde) || !currentSerdePair.getValue().equals(valueSerde)) { throw new IllegalArgumentException(String.format("Serde for table %s is already defined. Cannot change it to " + "different serdes.", tableId)); } return streamSerdes.get(tableId); }
public static <K, V> KVSerde<K, V> of(Serde<K> keySerde, Serde<V> valueSerde) { return new KVSerde<>(keySerde, valueSerde); }
/** * Construct a table manager instance * @param config job configuration * @param serdes Serde instances for tables */ public TableManager(Config config, Map<String, Serde<Object>> serdes) { new JavaTableConfig(config).getTableIds().forEach(tableId -> { // Construct the table provider String tableProviderFactory = config.get(String.format(JavaTableConfig.TABLE_PROVIDER_FACTORY, tableId)); // Construct the KVSerde JavaTableConfig tableConfig = new JavaTableConfig(config); KVSerde serde = KVSerde.of( serdes.get(tableConfig.getKeySerde(tableId)), serdes.get(tableConfig.getValueSerde(tableId))); TableSpec tableSpec = new TableSpec(tableId, serde, tableProviderFactory, config.subset(String.format(JavaTableConfig.TABLE_ID_PREFIX, tableId) + ".")); addTable(tableSpec); logger.info("Added table " + tableSpec.getId()); }); }
KV<Serde, Serde> getOrCreateTableSerdes(String tableId, KVSerde kvSerde) { Serde keySerde, valueSerde; keySerde = kvSerde.getKeySerde(); valueSerde = kvSerde.getValueSerde(); if (!tableSerdes.containsKey(tableId)) { tableSerdes.put(tableId, KV.of(keySerde, valueSerde)); return tableSerdes.get(tableId); } KV<Serde, Serde> currentSerdePair = tableSerdes.get(tableId); if (!currentSerdePair.getKey().equals(keySerde) || !currentSerdePair.getValue().equals(valueSerde)) { throw new IllegalArgumentException(String.format("Serde for table %s is already defined. Cannot change it to " + "different serdes.", tableId)); } return streamSerdes.get(tableId); }
public static <K, V> KVSerde<K, V> of(Serde<K> keySerde, Serde<V> valueSerde) { return new KVSerde<>(keySerde, valueSerde); }
/** * Construct a table manager instance * @param config job configuration * @param serdes Serde instances for tables */ public TableManager(Config config, Map<String, Serde<Object>> serdes) { new JavaTableConfig(config).getTableIds().forEach(tableId -> { // Construct the table provider String tableProviderFactory = config.get(String.format(JavaTableConfig.TABLE_PROVIDER_FACTORY, tableId)); // Construct the KVSerde JavaTableConfig tableConfig = new JavaTableConfig(config); KVSerde serde = KVSerde.of( serdes.get(tableConfig.getKeySerde(tableId)), serdes.get(tableConfig.getValueSerde(tableId))); TableSpec tableSpec = new TableSpec(tableId, serde, tableProviderFactory, config.subset(String.format(JavaTableConfig.TABLE_ID_PREFIX, tableId) + ".")); addTable(tableSpec); logger.info("Added table " + tableSpec.getId()); }); }
KV<Serde, Serde> getOrCreateTableSerdes(String tableId, KVSerde kvSerde) { Serde keySerde, valueSerde; keySerde = kvSerde.getKeySerde(); valueSerde = kvSerde.getValueSerde(); if (!tableSerdes.containsKey(tableId)) { tableSerdes.put(tableId, KV.of(keySerde, valueSerde)); return tableSerdes.get(tableId); } KV<Serde, Serde> currentSerdePair = tableSerdes.get(tableId); if (!currentSerdePair.getKey().equals(keySerde) || !currentSerdePair.getValue().equals(valueSerde)) { throw new IllegalArgumentException(String.format("Serde for table %s is already defined. Cannot change it to " + "different serdes.", tableId)); } return streamSerdes.get(tableId); }
private InMemoryTableDescriptor createTableDescriptor() { return new InMemoryTableDescriptor(TABLE_ID, new KVSerde(new NoOpSerde(), new NoOpSerde())); } }
/** * Construct a table manager instance * @param config job configuration * @param serdes Serde instances for tables */ public TableManager(Config config, Map<String, Serde<Object>> serdes) { new JavaTableConfig(config).getTableIds().forEach(tableId -> { // Construct the table provider String tableProviderFactory = config.get(String.format(JavaTableConfig.TABLE_PROVIDER_FACTORY, tableId)); // Construct the KVSerde JavaTableConfig tableConfig = new JavaTableConfig(config); KVSerde serde = KVSerde.of( serdes.get(tableConfig.getKeySerde(tableId)), serdes.get(tableConfig.getValueSerde(tableId))); TableSpec tableSpec = new TableSpec(tableId, serde, tableProviderFactory, config.subset(String.format(JavaTableConfig.TABLE_ID_PREFIX, tableId) + ".")); addTable(tableSpec); logger.info("Added table " + tableSpec.getId()); }); }
KV<Serde, Serde> getOrCreateTableSerdes(String tableId, KVSerde kvSerde) { Serde keySerde, valueSerde; keySerde = kvSerde.getKeySerde(); valueSerde = kvSerde.getValueSerde(); if (!tableSerdes.containsKey(tableId)) { tableSerdes.put(tableId, KV.of(keySerde, valueSerde)); return tableSerdes.get(tableId); } KV<Serde, Serde> currentSerdePair = tableSerdes.get(tableId); if (!currentSerdePair.getKey().equals(keySerde) || !currentSerdePair.getValue().equals(valueSerde)) { throw new IllegalArgumentException(String.format("Serde for table %s is already defined. Cannot change it to " + "different serdes.", tableId)); } return streamSerdes.get(tableId); }
private RocksDbTableDescriptor createTableDescriptor() { return new RocksDbTableDescriptor(TABLE_ID, new KVSerde(new NoOpSerde(), new NoOpSerde())); } }
/** * Construct a table manager instance * @param config job configuration * @param serdes Serde instances for tables */ public TableManager(Config config, Map<String, Serde<Object>> serdes) { new JavaTableConfig(config).getTableIds().forEach(tableId -> { // Construct the table provider String tableProviderFactory = config.get(String.format(JavaTableConfig.TABLE_PROVIDER_FACTORY, tableId)); // Construct the KVSerde JavaTableConfig tableConfig = new JavaTableConfig(config); KVSerde serde = KVSerde.of( serdes.get(tableConfig.getKeySerde(tableId)), serdes.get(tableConfig.getValueSerde(tableId))); TableSpec tableSpec = new TableSpec(tableId, serde, tableProviderFactory, config.subset(String.format(JavaTableConfig.TABLE_ID_PREFIX, tableId) + ".")); addTable(tableSpec); logger.info("Added table " + tableSpec.getId()); }); }
KV<Serde, Serde> getOrCreateTableSerdes(String tableId, KVSerde kvSerde) { Serde keySerde, valueSerde; keySerde = kvSerde.getKeySerde(); valueSerde = kvSerde.getValueSerde(); if (!tableSerdes.containsKey(tableId)) { tableSerdes.put(tableId, KV.of(keySerde, valueSerde)); return tableSerdes.get(tableId); } KV<Serde, Serde> currentSerdePair = tableSerdes.get(tableId); if (!currentSerdePair.getKey().equals(keySerde) || !currentSerdePair.getValue().equals(valueSerde)) { throw new IllegalArgumentException(String.format("Serde for table %s is already defined. Cannot change it to " + "different serdes.", tableId)); } return streamSerdes.get(tableId); }
private LocalTableDescriptor createTableDescriptor() { return new MockLocalTableDescriptor(TABLE_ID, new KVSerde(new StringSerde(), new IntegerSerde())); }
/** * Constructs an {@link OutputDescriptor} instance. * * @param streamId id of the stream * @param namespace namespace for the Event Hubs entity to produce to, not null * @param entityPath entity path for the Event Hubs entity to produce to, not null * @param valueSerde serde the values in the messages in the stream * @param systemDescriptor system descriptor this stream descriptor was obtained from */ EventHubsOutputDescriptor(String streamId, String namespace, String entityPath, Serde valueSerde, SystemDescriptor systemDescriptor) { super(streamId, KVSerde.of(new NoOpSerde<>(), valueSerde), systemDescriptor); this.namespace = StringUtils.stripToNull(namespace); this.entityPath = StringUtils.stripToNull(entityPath); if (this.namespace == null || this.entityPath == null) { throw new ConfigException(String.format("Missing namespace and entity path Event Hubs output descriptor in " // + "system: {%s}, stream: {%s}", getSystemName(), streamId)); } }
KV<Serde, Serde> getOrCreateStreamSerdes(String streamId, Serde serde) { Serde keySerde, valueSerde; KV<Serde, Serde> currentSerdePair = streamSerdes.get(streamId); if (serde instanceof KVSerde) { keySerde = ((KVSerde) serde).getKeySerde(); valueSerde = ((KVSerde) serde).getValueSerde(); } else { keySerde = new NoOpSerde(); valueSerde = serde; } if (currentSerdePair == null) { if (keySerde instanceof NoOpSerde) { LOGGER.info("Using NoOpSerde as the key serde for stream " + streamId + ". Keys will not be (de)serialized"); } if (valueSerde instanceof NoOpSerde) { LOGGER.info("Using NoOpSerde as the value serde for stream " + streamId + ". Values will not be (de)serialized"); } streamSerdes.put(streamId, KV.of(keySerde, valueSerde)); } else if (!currentSerdePair.getKey().equals(keySerde) || !currentSerdePair.getValue().equals(valueSerde)) { throw new IllegalArgumentException(String.format("Serde for streamId: %s is already defined. Cannot change it to " + "different serdes.", streamId)); } return streamSerdes.get(streamId); }
intermediateStream1, new KVSerde<>(new NoOpSerde(), new NoOpSerde())); GenericOutputDescriptor<KV<Object, Object>> intermediateOutput1 = system1Descriptor.getOutputDescriptor( intermediateStream1, new KVSerde<>(new NoOpSerde(), new NoOpSerde())); intermediateBroadcast, new KVSerde<>(new NoOpSerde<>(), new NoOpSerde<>())); GenericOutputDescriptor<KV<Object, Object>> intermediateBroacastOutput1 = system1Descriptor.getOutputDescriptor( intermediateBroadcast, new KVSerde<>(new NoOpSerde<>(), new NoOpSerde<>())); inputDescriptors.put(intermediateStream1, intermediateInput1); outputDescriptors.put(intermediateStream1, intermediateOutput1);
/** * Constructs an {@link InputDescriptor} instance. * * @param streamId id of the stream * @param namespace namespace for the Event Hubs entity to consume from, not null * @param entityPath entity path for the Event Hubs entity to consume from, not null * @param valueSerde serde the values in the messages in the stream * @param systemDescriptor system descriptor this stream descriptor was obtained from */ EventHubsInputDescriptor(String streamId, String namespace, String entityPath, Serde valueSerde, SystemDescriptor systemDescriptor) { super(streamId, KVSerde.of(new NoOpSerde<>(), valueSerde), systemDescriptor, null); this.namespace = StringUtils.stripToNull(namespace); this.entityPath = StringUtils.stripToNull(entityPath); if (this.namespace == null || this.entityPath == null) { throw new ConfigException(String.format("Missing namespace and entity path Event Hubs input descriptor in " // + "system: {%s}, stream: {%s}", getSystemName(), streamId)); } }
KV<Serde, Serde> getOrCreateStreamSerdes(String streamId, Serde serde) { Serde keySerde, valueSerde; KV<Serde, Serde> currentSerdePair = streamSerdes.get(streamId); if (serde instanceof KVSerde) { keySerde = ((KVSerde) serde).getKeySerde(); valueSerde = ((KVSerde) serde).getValueSerde(); } else { keySerde = new NoOpSerde(); valueSerde = serde; } if (currentSerdePair == null) { if (keySerde instanceof NoOpSerde) { LOGGER.info("Using NoOpSerde as the key serde for stream " + streamId + ". Keys will not be (de)serialized"); } if (valueSerde instanceof NoOpSerde) { LOGGER.info("Using NoOpSerde as the value serde for stream " + streamId + ". Values will not be (de)serialized"); } streamSerdes.put(streamId, KV.of(keySerde, valueSerde)); } else if (!currentSerdePair.getKey().equals(keySerde) || !currentSerdePair.getValue().equals(valueSerde)) { throw new IllegalArgumentException(String.format("Serde for streamId: %s is already defined. Cannot change it to " + "different serdes.", streamId)); } return streamSerdes.get(streamId); }