/** * Creates a coordinator stream kafka topic. */ private void createCoordinatorStream() { SystemAdmin coordinatorSystemAdmin = null; SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); coordinatorSystemAdmin = systemAdmins.getSystemAdmin(coordinatorSystemStream.getSystem()); String streamName = coordinatorSystemStream.getStream(); StreamSpec coordinatorSpec = StreamSpec.createCoordinatorStreamSpec(streamName, coordinatorSystemStream.getSystem()); if (coordinatorSystemAdmin.createStream(coordinatorSpec)) { LOG.info("Created coordinator stream: {}.", streamName); } else { LOG.info("Coordinator stream: {} already exists.", streamName); } }
public CoordinatorStreamSystemProducer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemProducer systemProducer = systemFactory.getProducer(coordinatorSystemStream.getSystem(), config, registry); this.systemStream = coordinatorSystemStream; this.systemProducer = systemProducer; this.systemAdmin = systemAdmin; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
public CoordinatorStreamSystemProducer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemProducer systemProducer = systemFactory.getProducer(coordinatorSystemStream.getSystem(), config, registry); this.systemStream = coordinatorSystemStream; this.systemProducer = systemProducer; this.systemAdmin = systemAdmin; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
public CoordinatorStreamSystemProducer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemProducer systemProducer = systemFactory.getProducer(coordinatorSystemStream.getSystem(), config, registry); this.systemStream = coordinatorSystemStream; this.systemProducer = systemProducer; this.systemAdmin = systemAdmin; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
@Before public void setup() { mockCoordinatorStreamSystemFactory = new MockCoordinatorStreamSystemFactory(); MockCoordinatorStreamSystemFactory.enableMockConsumerCache(); PowerMockito.mockStatic(CoordinatorStreamUtil.class); when(CoordinatorStreamUtil.getCoordinatorSystemFactory(anyObject())).thenReturn(mockCoordinatorStreamSystemFactory); when(CoordinatorStreamUtil.getCoordinatorSystemStream(anyObject())).thenReturn(new SystemStream("test-kafka", "test")); when(CoordinatorStreamUtil.getCoordinatorStreamName(anyObject(), anyObject())).thenReturn("test"); }
public CoordinatorStreamSystemProducer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemProducer systemProducer = systemFactory.getProducer(coordinatorSystemStream.getSystem(), config, registry); this.systemStream = coordinatorSystemStream; this.systemProducer = systemProducer; this.systemAdmin = systemAdmin; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
public CoordinatorStreamSystemProducer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemProducer systemProducer = systemFactory.getProducer(coordinatorSystemStream.getSystem(), config, registry); this.systemStream = coordinatorSystemStream; this.systemProducer = systemProducer; this.systemAdmin = systemAdmin; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
public CoordinatorStreamSystemConsumer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemConsumer systemConsumer = systemFactory.getConsumer(coordinatorSystemStream.getSystem(), config, registry); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); this.systemConsumer = systemConsumer; this.systemAdmin = systemAdmin; this.configMap = new HashMap<>(); this.isBootstrapped = false; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
public CoordinatorStreamSystemConsumer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemConsumer systemConsumer = systemFactory.getConsumer(coordinatorSystemStream.getSystem(), config, registry); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); this.systemConsumer = systemConsumer; this.systemAdmin = systemAdmin; this.configMap = new HashMap<>(); this.isBootstrapped = false; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
@Before public void setUp() throws NoSuchFieldException, NoSuchMethodException { configMap = new HashMap<>(); configMap.put("job.name", "test-job"); configMap.put("job.coordinator.system", "kafka"); configMap.put("task.inputs", "kafka.topic1"); configMap.put("systems.kafka.samza.factory", "org.apache.samza.system.MockSystemFactory"); configMap.put("samza.cluster-manager.factory", "org.apache.samza.clustermanager.MockClusterResourceManagerFactory"); configMap.put("job.coordinator.monitor-partition-change.frequency.ms", "1"); MockSystemFactory.MSG_QUEUES.put(new SystemStreamPartition("kafka", "topic1", new Partition(0)), new ArrayList<>()); MockSystemFactory.MSG_QUEUES.put(new SystemStreamPartition("kafka", "__samza_coordinator_test-job_1", new Partition(0)), new ArrayList<>()); MockCoordinatorStreamSystemFactory.enableMockConsumerCache(); PowerMockito.mockStatic(CoordinatorStreamUtil.class); when(CoordinatorStreamUtil.getCoordinatorSystemFactory(anyObject())).thenReturn(new MockCoordinatorStreamSystemFactory()); when(CoordinatorStreamUtil.getCoordinatorSystemStream(anyObject())).thenReturn(new SystemStream("kafka", "test")); when(CoordinatorStreamUtil.getCoordinatorStreamName(anyObject(), anyObject())).thenReturn("test"); }
public CoordinatorStreamSystemConsumer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemConsumer systemConsumer = systemFactory.getConsumer(coordinatorSystemStream.getSystem(), config, registry); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); this.systemConsumer = systemConsumer; this.systemAdmin = systemAdmin; this.configMap = new HashMap<>(); this.isBootstrapped = false; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
public CoordinatorStreamSystemConsumer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemConsumer systemConsumer = systemFactory.getConsumer(coordinatorSystemStream.getSystem(), config, registry); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); this.systemConsumer = systemConsumer; this.systemAdmin = systemAdmin; this.configMap = new HashMap<>(); this.isBootstrapped = false; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
public CoordinatorStreamSystemConsumer(Config config, MetricsRegistry registry) { SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); SystemAdmin systemAdmin = systemFactory.getAdmin(coordinatorSystemStream.getSystem(), config); SystemConsumer systemConsumer = systemFactory.getConsumer(coordinatorSystemStream.getSystem(), config, registry); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); this.systemConsumer = systemConsumer; this.systemAdmin = systemAdmin; this.configMap = new HashMap<>(); this.isBootstrapped = false; this.keySerde = new JsonSerde<>(); this.messageSerde = new JsonSerde<>(); }
@Before public void setup() { MockCoordinatorStreamSystemFactory.enableMockConsumerCache(); mockCoordinatorStreamSystemFactory = new MockCoordinatorStreamSystemFactory(); PowerMockito.mockStatic(CoordinatorStreamUtil.class); when(CoordinatorStreamUtil.getCoordinatorSystemFactory(anyObject())).thenReturn(mockCoordinatorStreamSystemFactory); when(CoordinatorStreamUtil.getCoordinatorSystemStream(anyObject())).thenReturn(new SystemStream("test-kafka", "test")); when(CoordinatorStreamUtil.getCoordinatorStreamName(anyObject(), anyObject())).thenReturn("test"); }
@Before public void setUp() { MockCoordinatorStreamSystemFactory.enableMockConsumerCache(); Map<String, String> configMap = ImmutableMap.of("job.name", "test-job", "job.coordinator.system", "test-kafka"); MockCoordinatorStreamSystemFactory systemFactory = new MockCoordinatorStreamSystemFactory(); PowerMockito.mockStatic(CoordinatorStreamUtil.class); when(CoordinatorStreamUtil.getCoordinatorSystemFactory(anyObject())).thenReturn(systemFactory); when(CoordinatorStreamUtil.getCoordinatorSystemStream(anyObject())).thenReturn(new SystemStream("test-kafka", "test")); when(CoordinatorStreamUtil.getCoordinatorStreamName(anyObject(), anyObject())).thenReturn("test"); coordinatorStreamStore = new CoordinatorStreamStore(SetTaskContainerMapping.TYPE, new MapConfig(configMap), new MetricsRegistryMap()); coordinatorStreamStore.init(); }
public CoordinatorStreamStore(String namespace, Config config, MetricsRegistry metricsRegistry) { this.config = config; this.type = namespace; this.keySerde = new CoordinatorStreamKeySerde(type); this.coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); this.systemProducer = systemFactory.getProducer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemConsumer = systemFactory.getConsumer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemAdmin = systemFactory.getAdmin(this.coordinatorSystemStream.getSystem(), config); }
public CoordinatorStreamStore(String namespace, Config config, MetricsRegistry metricsRegistry) { this.config = config; this.type = namespace; this.keySerde = new JsonSerde<>(); this.coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); this.systemProducer = systemFactory.getProducer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemConsumer = systemFactory.getConsumer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemAdmin = systemFactory.getAdmin(this.coordinatorSystemStream.getSystem(), config); }
public CoordinatorStreamStore(String namespace, Config config, MetricsRegistry metricsRegistry) { this.config = config; this.type = namespace; this.keySerde = new JsonSerde<>(); this.coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); this.systemProducer = systemFactory.getProducer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemConsumer = systemFactory.getConsumer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemAdmin = systemFactory.getAdmin(this.coordinatorSystemStream.getSystem(), config); }
public CoordinatorStreamStore(String namespace, Config config, MetricsRegistry metricsRegistry) { this.config = config; this.type = namespace; this.keySerde = new JsonSerde<>(); this.coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); this.systemProducer = systemFactory.getProducer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemConsumer = systemFactory.getConsumer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemAdmin = systemFactory.getAdmin(this.coordinatorSystemStream.getSystem(), config); }
public CoordinatorStreamStore(String namespace, Config config, MetricsRegistry metricsRegistry) { this.config = config; this.type = namespace; this.keySerde = new JsonSerde<>(); this.coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); this.coordinatorSystemStreamPartition = new SystemStreamPartition(coordinatorSystemStream, new Partition(0)); SystemFactory systemFactory = CoordinatorStreamUtil.getCoordinatorSystemFactory(config); this.systemProducer = systemFactory.getProducer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemConsumer = systemFactory.getConsumer(this.coordinatorSystemStream.getSystem(), config, metricsRegistry); this.systemAdmin = systemFactory.getAdmin(this.coordinatorSystemStream.getSystem(), config); }