public ConfigBasedIOResolver(Config config) { this.config = config; String metadataTopicPrefix = config.get(CFG_METADATA_TOPIC_PREFIX, DEFAULT_METADATA_TOPIC_PREFIX); this.changeLogStorePrefix = metadataTopicPrefix + (metadataTopicPrefix.isEmpty() ? "" : "_"); }
@Override public void init(Context context) throws Exception { Config config = context.getJobContext().getConfig(); this.expectedMessageCount = config.getInt("app.messageCount"); this.outputTopic = config.get("app.outputTopic", "output"); this.outputSystem = config.get("app.outputSystem", "test-system"); }
public short getShort(String k, short defaultValue) { if (containsKey(k)) return Short.parseShort(get(k)); else return defaultValue; }
public long getLong(String k, long defaultValue) { if (containsKey(k)) return Long.parseLong(get(k)); else return defaultValue; }
public int getInt(String k, int defaultValue) { if (containsKey(k)) return Integer.parseInt(get(k)); else return defaultValue; }
public double getDouble(String k, double defaultValue) { if (containsKey(k)) return Double.parseDouble(get(k)); else return defaultValue; }
public List<String> getList(String k, List<String> defaultValue) { if (!containsKey(k)) return defaultValue; String value = get(k); if (value.trim().isEmpty()) { return defaultValue; } String[] pieces = value.split("\\s*,\\s*"); return Arrays.asList(pieces); }
public Date getDate(String k, SimpleDateFormat format, Date defaultValue) { if (!containsKey(k)) return defaultValue; try { return format.parse(get(k)); } catch (ParseException e) { throw new ConfigException("Date format exception.", e); } }
public TestRemoteStoreIOResolver(Config config) { this.config = config; String metadataTopicPrefix = config.get(CFG_METADATA_TOPIC_PREFIX, DEFAULT_METADATA_TOPIC_PREFIX); this.changeLogStorePrefix = metadataTopicPrefix + (metadataTopicPrefix.isEmpty() ? "" : "_"); }
public double getDouble(String k) { if (containsKey(k)) return Double.parseDouble(get(k)); else throw new ConfigException("Missing key " + k + "."); }
public boolean getBoolean(String k) { if (containsKey(k)) return "true".equalsIgnoreCase(get(k)); else throw new ConfigException("Missing key " + k + "."); }
@Override public TaskNameGrouper build(Config config) { return new SingleContainerGrouper(config.get(JobConfig.PROCESSOR_ID())); } }
private SystemStream getCoordinatorSystemStream(Config config) { assertNotNull(config.get("job.coordinator.system")); assertNotNull(config.get("job.name")); return new SystemStream(config.get("job.coordinator.system"), CoordinatorStreamUtil.getCoordinatorStreamName(config.get("job.name"), config.get("job.id") == null ? "1" : config.get("job.id"))); }
@SuppressWarnings("unchecked") public <T> Class<T> getClass(String k) { if (containsKey(k)) { try { return (Class<T>) Class.forName(get(k)); } catch (Exception e) { throw new ConfigException("Unable to find class.", e); } } else { throw new ConfigException("Missing key " + k + "."); } }
public Date getDate(String k, SimpleDateFormat format) { if (!containsKey(k)) throw new ConfigException("Missing key " + k + "."); try { return format.parse(get(k)); } catch (ParseException e) { throw new ConfigException("Date format exception.", e); } }
private void validateIntermediateStreamConfigure(String streamId, String physicalName, Config config) { Config intStreamConfig = config.subset(String.format("streams.%s.", streamId), true); assertEquals("intermediate-system", intStreamConfig.get("samza.system")); assertEquals(String.valueOf(Integer.MAX_VALUE), intStreamConfig.get("samza.priority")); assertEquals("true", intStreamConfig.get("samza.delete.committed.messages")); assertEquals(physicalName, intStreamConfig.get("samza.physical.name")); assertEquals("true", intStreamConfig.get("samza.intermediate")); assertEquals("oldest", intStreamConfig.get("samza.offset.default")); }
private void validateStreamSerdeConfigure(String streamId, Config config, Map<String, Serde> deserializedSerdes) { Config streamConfig = config.subset(String.format("streams.%s.samza.", streamId)); String keySerdeName = streamConfig.get("key.serde"); String valueSerdeName = streamConfig.get("msg.serde"); assertTrue(String.format("Serialized serdes should contain %s key serde", streamId), deserializedSerdes.containsKey(keySerdeName)); assertTrue(String.format("Serialized %s key serde should be a StringSerde", streamId), keySerdeName.startsWith(StringSerde.class.getSimpleName())); assertTrue(String.format("Serialized serdes should contain %s msg serde", streamId), deserializedSerdes.containsKey(valueSerdeName)); assertTrue(String.format("Serialized %s msg serde should be a JsonSerdeV2", streamId), valueSerdeName.startsWith(JsonSerdeV2.class.getSimpleName())); }
private void validateTableSerdeConfigure(String tableId, Config config, Map<String, Serde> deserializedSerdes) { Config streamConfig = config.subset(String.format("stores.%s.", tableId)); String keySerdeName = streamConfig.get("key.serde"); String valueSerdeName = streamConfig.get("msg.serde"); assertTrue(String.format("Serialized serdes should contain %s key serde", tableId), deserializedSerdes.containsKey(keySerdeName)); assertTrue(String.format("Serialized %s key serde should be a StringSerde", tableId), keySerdeName.startsWith(StringSerde.class.getSimpleName())); assertTrue(String.format("Serialized serdes should contain %s value serde", tableId), deserializedSerdes.containsKey(valueSerdeName)); assertTrue(String.format("Serialized %s msg serde should be a JsonSerdeV2", tableId), valueSerdeName.startsWith(JsonSerdeV2.class.getSimpleName())); }
public MockCoordinatorStreamSystemConsumer getCoordinatorStreamSystemConsumer(Config config, MetricsRegistry registry) { return new MockCoordinatorStreamSystemConsumer(getCoordinatorSystemStream(config), getConsumer(config.get("job.coordinator.system"), config, registry), getAdmin(config.get("job.coordinator.system"), config)); }
public MockCoordinatorStreamSystemProducer getCoordinatorStreamSystemProducer(Config config, MetricsRegistry registry) { return new MockCoordinatorStreamSystemProducer(getCoordinatorSystemStream(config), getProducer(config.get("job.coordinator.system"), config, registry), getAdmin(config.get("job.coordinator.system"), config)); }