public void createStreams(List<StreamSpec> streams) { Multimap<String, StreamSpec> streamsGroupedBySystem = HashMultimap.create(); streams.forEach(streamSpec -> streamsGroupedBySystem.put(streamSpec.getSystemName(), streamSpec)); for (Map.Entry<String, Collection<StreamSpec>> entry : streamsGroupedBySystem.asMap().entrySet()) { String systemName = entry.getKey(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); for (StreamSpec stream : entry.getValue()) { LOGGER.info("Creating stream {} with partitions {} on system {}", new Object[]{stream.getPhysicalName(), stream.getPartitionCount(), systemName}); systemAdmin.createStream(stream); } } }
public void createStreams(List<StreamSpec> streams) { Multimap<String, StreamSpec> streamsGroupedBySystem = HashMultimap.create(); streams.forEach(streamSpec -> streamsGroupedBySystem.put(streamSpec.getSystemName(), streamSpec)); for (Map.Entry<String, Collection<StreamSpec>> entry : streamsGroupedBySystem.asMap().entrySet()) { String systemName = entry.getKey(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); for (StreamSpec stream : entry.getValue()) { LOGGER.info("Creating stream {} with partitions {} on system {}", new Object[]{stream.getPhysicalName(), stream.getPartitionCount(), systemName}); systemAdmin.createStream(stream); } } }
public void createStreams(List<StreamSpec> streams) { Multimap<String, StreamSpec> streamsGroupedBySystem = HashMultimap.create(); streams.forEach(streamSpec -> streamsGroupedBySystem.put(streamSpec.getSystemName(), streamSpec)); for (Map.Entry<String, Collection<StreamSpec>> entry : streamsGroupedBySystem.asMap().entrySet()) { String systemName = entry.getKey(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); for (StreamSpec stream : entry.getValue()) { LOGGER.info("Creating stream {} with partitions {} on system {}", new Object[]{stream.getPhysicalName(), stream.getPartitionCount(), systemName}); systemAdmin.createStream(stream); } } }
public void createStreams(List<StreamSpec> streams) { Multimap<String, StreamSpec> streamsGroupedBySystem = HashMultimap.create(); streams.forEach(streamSpec -> streamsGroupedBySystem.put(streamSpec.getSystemName(), streamSpec)); for (Map.Entry<String, Collection<StreamSpec>> entry : streamsGroupedBySystem.asMap().entrySet()) { String systemName = entry.getKey(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); for (StreamSpec stream : entry.getValue()) { LOGGER.info("Creating stream {} with partitions {} on system {}", new Object[]{stream.getPhysicalName(), stream.getPartitionCount(), systemName}); systemAdmin.createStream(stream); } } }
public void createStreams(List<StreamSpec> streams) { Multimap<String, StreamSpec> streamsGroupedBySystem = HashMultimap.create(); streams.forEach(streamSpec -> streamsGroupedBySystem.put(streamSpec.getSystemName(), streamSpec)); for (Map.Entry<String, Collection<StreamSpec>> entry : streamsGroupedBySystem.asMap().entrySet()) { String systemName = entry.getKey(); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(systemName); for (StreamSpec stream : entry.getValue()) { LOGGER.info("Creating stream {} with partitions {} on system {}", new Object[]{stream.getPhysicalName(), stream.getPartitionCount(), systemName}); systemAdmin.createStream(stream); } } }
/** * Creates a coordinator stream kafka topic. */ private void createCoordinatorStream() { SystemAdmin coordinatorSystemAdmin = null; SystemStream coordinatorSystemStream = CoordinatorStreamUtil.getCoordinatorSystemStream(config); coordinatorSystemAdmin = systemAdmins.getSystemAdmin(coordinatorSystemStream.getSystem()); String streamName = coordinatorSystemStream.getStream(); StreamSpec coordinatorSpec = StreamSpec.createCoordinatorStreamSpec(streamName, coordinatorSystemStream.getSystem()); if (coordinatorSystemAdmin.createStream(coordinatorSpec)) { LOG.info("Created coordinator stream: {}.", streamName); } else { LOG.info("Coordinator stream: {} already exists.", streamName); } }
@Test public void testCreateCoordinatorStream() { SystemAdmin admin = Mockito.spy(systemAdmin()); StreamSpec spec = StreamSpec.createCoordinatorStreamSpec("testCoordinatorStream", "testSystem"); admin.createStream(spec); admin.validateStream(spec); Mockito.verify(admin).createStream(Mockito.any()); }
@Test public void testCreateStreams() { StreamSpec spec1 = new StreamSpec(STREAM1, STREAM1, SYSTEM1); StreamSpec spec2 = new StreamSpec(STREAM2, STREAM2, SYSTEM2); List<StreamSpec> specList = new ArrayList<>(); specList.add(spec1); specList.add(spec2); SystemAdmin admin1 = mock(SystemAdmin.class); SystemAdmin admin2 = mock(SystemAdmin.class); SystemAdmins systemAdmins = mock(SystemAdmins.class); when(systemAdmins.getSystemAdmin(SYSTEM1)).thenReturn(admin1); when(systemAdmins.getSystemAdmin(SYSTEM2)).thenReturn(admin2); StreamManager manager = new StreamManager(systemAdmins); manager.createStreams(specList); ArgumentCaptor<StreamSpec> captor = ArgumentCaptor.forClass(StreamSpec.class); verify(admin1).createStream(captor.capture()); assertEquals(STREAM1, captor.getValue().getPhysicalName()); captor = ArgumentCaptor.forClass(StreamSpec.class); verify(admin2).createStream(captor.capture()); assertEquals(STREAM2, captor.getValue().getPhysicalName()); }
if (systemAdmin.createStream(changelogSpec)) { LOG.info(String.format("created changelog stream %s.", systemStream.getStream())); } else { StreamSpec accesslogSpec = new StreamSpec(accesslogStream, accesslogStream, systemStream.getSystem(), maxChangeLogStreamPartitions); systemAdmin.createStream(accesslogSpec); systemAdmin.validateStream(accesslogSpec);
if (systemAdmin.createStream(changelogSpec)) { LOG.info(String.format("created changelog stream %s.", systemStream.getStream())); } else { StreamSpec accesslogSpec = new StreamSpec(accesslogStream, accesslogStream, systemStream.getSystem(), maxChangeLogStreamPartitions); systemAdmin.createStream(accesslogSpec); systemAdmin.validateStream(accesslogSpec);
if (systemAdmin.createStream(changelogSpec)) { LOG.info(String.format("created changelog stream %s.", systemStream.getStream())); } else { StreamSpec accesslogSpec = new StreamSpec(accesslogStream, accesslogStream, systemStream.getSystem(), maxChangeLogStreamPartitions); systemAdmin.createStream(accesslogSpec); systemAdmin.validateStream(accesslogSpec);
if (systemAdmin.createStream(changelogSpec)) { LOG.info(String.format("created changelog stream %s.", systemStream.getStream())); } else { StreamSpec accesslogSpec = new StreamSpec(accesslogStream, accesslogStream, systemStream.getSystem(), maxChangeLogStreamPartitions); systemAdmin.createStream(accesslogSpec); systemAdmin.validateStream(accesslogSpec);
if (systemAdmin.createStream(changelogSpec)) { LOG.info(String.format("created changelog stream %s.", systemStream.getStream())); } else { StreamSpec accesslogSpec = new StreamSpec(accesslogStream, accesslogStream, systemStream.getSystem(), maxChangeLogStreamPartitions); systemAdmin.createStream(accesslogSpec); systemAdmin.validateStream(accesslogSpec);
@Test(expected = TopicAlreadyMarkedForDeletionException.class) public void testStartFailsOnTopicCreationErrors() { KafkaStreamSpec checkpointSpec = new KafkaStreamSpec(CHECKPOINT_TOPIC, CHECKPOINT_TOPIC, CHECKPOINT_SYSTEM, 1); // create an admin that throws an exception during createStream SystemAdmin mockAdmin = newAdmin("0", "10"); doThrow(new TopicAlreadyMarkedForDeletionException("invalid stream")).when(mockAdmin).createStream(checkpointSpec); SystemFactory factory = newFactory(mock(SystemProducer.class), mock(SystemConsumer.class), mockAdmin); KafkaCheckpointManager checkpointManager = new KafkaCheckpointManager(checkpointSpec, factory, true, mock(Config.class), mock(MetricsRegistry.class), null, new KafkaCheckpointLogKeySerde()); // expect an exception during startup checkpointManager.createResources(); checkpointManager.start(); }
public TestInMemorySystem() { Config config = new MapConfig(); systemFactory = new InMemorySystemFactory(); systemAdmin = systemFactory.getAdmin(SYSTEM_NAME, config); systemAdmin.createStream(new StreamSpec(STREAM_NAME, STREAM_NAME, SYSTEM_NAME, PARTITION_COUNT)); }
systemAdmin.createStream(streamSpec); systemAdmin.stop();
systemAdmin.createStream(streamSpec); systemAdmin.stop();