private KafkaOffsetBackingStore createKafkaOffsetBackingStore(WorkerConfig workerConfig) { KafkaOffsetBackingStore kafkaOffsetBackingStore = new KafkaOffsetBackingStore(); kafkaOffsetBackingStore.configure(workerConfig); kafkaOffsetBackingStore.start(); return kafkaOffsetBackingStore; }
private static MirusOffsetTool newOffsetTool(Args args) throws IOException { // This needs to be the admin topic properties. // By default these are in the worker properties file, as this has the has admin producer and // consumer settings. Separating these might be wise - also useful for storing state in // source cluster if it proves necessary. final Map<String, String> properties = !args.propertiesFile.isEmpty() ? Utils.propsToStringMap(Utils.loadProps(args.propertiesFile)) : Collections.emptyMap(); final DistributedConfig config = new DistributedConfig(properties); final KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore(); offsetBackingStore.configure(config); // Avoid initializing the entire Kafka Connect plugin system by assuming the // internal.[key|value].converter is org.apache.kafka.connect.json.JsonConverter final Converter internalConverter = new JsonConverter(); internalConverter.configure(config.originalsWithPrefix("internal.key.converter."), true); final OffsetSetter offsetSetter = new OffsetSetter(internalConverter, offsetBackingStore); final OffsetFetcher offsetFetcher = new OffsetFetcher(config, internalConverter); final OffsetSerDe offsetSerDe = OffsetSerDeFactory.create(args.format); return new MirusOffsetTool(args, offsetFetcher, offsetSetter, offsetSerDe); }
public ConnectEmbedded(Properties workerConfig, Properties... connectorConfigs) throws Exception { Time time = new SystemTime(); DistributedConfig config = new DistributedConfig(Utils.propsToStringMap(workerConfig)); KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore(); offsetBackingStore.configure(config); //not sure if this is going to work but because we don't have advertised url we can get at least a fairly random String workerId = UUID.randomUUID().toString(); worker = new Worker(workerId, time, config, offsetBackingStore); StatusBackingStore statusBackingStore = new KafkaStatusBackingStore(time, worker.getInternalValueConverter()); statusBackingStore.configure(config); ConfigBackingStore configBackingStore = new KafkaConfigBackingStore(worker.getInternalValueConverter()); configBackingStore.configure(config); //advertisedUrl = "" as we don't have the rest server - hopefully this will not break anything herder = new DistributedHerder(config, time, worker, statusBackingStore, configBackingStore, ""); this.connectorConfigs = connectorConfigs; shutdownHook = new ShutdownHook(); }
String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort(); KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore(); offsetBackingStore.configure(config);
public KafkaConnectDistributed(Map<String, String> workerProps) { Time time = new SystemTime(); ConnectorFactory connectorFactory = new ConnectorFactory(); DistributedConfig config = new DistributedConfig(workerProps); rest = new RestServer(config); URI advertisedUrl = rest.advertisedUrl(); String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort(); KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore(); offsetBackingStore.configure(config); Worker worker = new Worker(workerId, time, connectorFactory, config, offsetBackingStore); StatusBackingStore statusBackingStore = new KafkaStatusBackingStore(time, worker.getInternalValueConverter()); statusBackingStore.configure(config); ConfigBackingStore configBackingStore = new KafkaConfigBackingStore(worker.getInternalValueConverter()); configBackingStore.configure(config); DistributedHerder herder = new DistributedHerder(config, time, worker, statusBackingStore, configBackingStore, advertisedUrl.toString()); connect = new Connect(herder, rest); try { connect.start(); } catch (Throwable t) { LOGGER.error("Failed to start Connector error", t); rest.stop(); connect.stop(); } }
workerConfig = new MemoryConfig(offsetBackingStoreProperties); } else if (StreamOptions.KAFKA_BACKING_STORE().getValue().equals(bs)) { offsetBackingStore = new KafkaOffsetBackingStore(); workerConfig = new KafkaConfig(offsetBackingStoreProperties); } else {
String workerId = advertisedUrl.getHost() + ":" + advertisedUrl.getPort(); KafkaOffsetBackingStore offsetBackingStore = new KafkaOffsetBackingStore(); offsetBackingStore.configure(configWithClientIdSuffix(workerProps, "offset"));