public CoordinatorStreamWriter(Config config) { coordinatorStreamSystemProducer = new CoordinatorStreamSystemProducer(config, new MetricsRegistryMap()); }
public CoordinatorStreamWriter(Config config) { coordinatorStreamSystemProducer = new CoordinatorStreamSystemProducer(config, new MetricsRegistryMap()); }
public CoordinatorStreamWriter(Config config) { coordinatorStreamSystemProducer = new CoordinatorStreamSystemProducer(config, new MetricsRegistryMap()); }
private Config getConfigFromPrevRun() { CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(userConfig, new MetricsRegistryMap()); consumer.register(); consumer.start(); consumer.bootstrap(); consumer.stop(); Config cfg = consumer.getConfig(); LOG.info("Previous config is: " + cfg.toString()); return cfg; } }
private Config getConfigFromPrevRun() { CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(userConfig, new MetricsRegistryMap()); consumer.register(); consumer.start(); consumer.bootstrap(); consumer.stop(); Config cfg = consumer.getConfig(); LOG.info("Previous config is: " + cfg.toString()); return cfg; } }
private Config getConfigFromPrevRun() { CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(userConfig, new MetricsRegistryMap()); consumer.register(); consumer.start(); consumer.bootstrap(); consumer.stop(); Config cfg = consumer.getConfig(); LOG.info("Previous config is: " + cfg.toString()); return cfg; } }
private Config getConfigFromPrevRun() { CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(userConfig, new MetricsRegistryMap()); consumer.register(); consumer.start(); consumer.bootstrap(); consumer.stop(); Config cfg = consumer.getConfig(); LOG.info("Previous config is: " + cfg.toString()); return cfg; } }
private Config getConfigFromPrevRun() { CoordinatorStreamSystemConsumer consumer = new CoordinatorStreamSystemConsumer(userConfig, new MetricsRegistryMap()); consumer.register(); consumer.start(); consumer.bootstrap(); consumer.stop(); Config cfg = consumer.getConfig(); LOG.info("Previous config is: " + cfg.toString()); return cfg; } }
@Before public void setup() { TaskInstanceMetrics metrics = new TaskInstanceMetrics("Partition 0", new MetricsRegistryMap()); listener = new TaskCallbackListener() { @Override public void onComplete(TaskCallback callback) { } @Override public void onFailure(TaskCallback callback, Throwable t) { } }; callbackManager = new TaskCallbackManager(listener, null, -1, 2, () -> System.nanoTime()); }
/** * Creates a instance of {@link SystemStreamPartitionMapper} using the stream partition expansion factory class * defined in the {@param config}. * @param config the configuration of the samza job. * @return the instantiated {@link SystemStreamPartitionMapper} object. */ private SystemStreamPartitionMapper getSystemStreamPartitionMapper(Config config) { JobConfig jobConfig = new JobConfig(config); String systemStreamPartitionMapperClass = jobConfig.getSystemStreamPartitionMapperFactoryName(); SystemStreamPartitionMapperFactory systemStreamPartitionMapperFactory = Util.getObj(systemStreamPartitionMapperClass, SystemStreamPartitionMapperFactory.class); return systemStreamPartitionMapperFactory.getStreamPartitionMapper(config, new MetricsRegistryMap()); }
/** * build the ContainerModels from job config file and put the results in the * map */ private void getContainerModels() { CoordinatorStreamManager coordinatorStreamManager = new CoordinatorStreamManager(jobConfig, new MetricsRegistryMap()); coordinatorStreamManager.register(getClass().getSimpleName()); coordinatorStreamManager.start(); coordinatorStreamManager.bootstrap(); ChangelogStreamManager changelogStreamManager = new ChangelogStreamManager(coordinatorStreamManager); JobModel jobModel = JobModelManager.apply(coordinatorStreamManager.getConfig(), changelogStreamManager.readPartitionMapping()).jobModel(); containers = jobModel.getContainers(); coordinatorStreamManager.stop(); }
@Test public void testTaskAssignmentManager() { TaskAssignmentManager taskAssignmentManager = new TaskAssignmentManager(config, new MetricsRegistryMap()); Map<String, String> expectedMap = ImmutableMap.of("Task0", "0", "Task1", "1", "Task2", "2", "Task3", "0", "Task4", "1"); for (Map.Entry<String, String> entry : expectedMap.entrySet()) { taskAssignmentManager.writeTaskContainerMapping(entry.getKey(), entry.getValue(), TaskMode.Active); } Map<String, String> localMap = taskAssignmentManager.readTaskAssignment(); assertEquals(expectedMap, localMap); taskAssignmentManager.close(); }
@Before public void setup() { this.context = new MockContext(); when(this.context.getTaskContext().getTaskMetricsRegistry()).thenReturn(new MetricsRegistryMap()); when(this.context.getTaskContext().getTaskModel()).thenReturn(mock(TaskModel.class)); when(this.context.getContainerContext().getContainerMetricsRegistry()).thenReturn(new MetricsRegistryMap()); }
@Test public void testTaskAssignmentManagerEmptyCoordinatorStream() { TaskAssignmentManager taskAssignmentManager = new TaskAssignmentManager(config, new MetricsRegistryMap()); Map<String, String> expectedMap = new HashMap<>(); Map<String, String> localMap = taskAssignmentManager.readTaskAssignment(); assertEquals(expectedMap, localMap); taskAssignmentManager.close(); } }
@Before public void setup() { this.context = new MockContext(); // individual tests can override this config if necessary when(this.context.getJobContext().getConfig()).thenReturn(mock(Config.class)); TaskModel taskModel = mock(TaskModel.class); when(taskModel.getTaskName()).thenReturn(new TaskName("task 0")); when(this.context.getTaskContext().getTaskModel()).thenReturn(taskModel); when(this.context.getTaskContext().getTaskMetricsRegistry()).thenReturn(new MetricsRegistryMap()); when(this.context.getContainerContext().getContainerMetricsRegistry()).thenReturn(new MetricsRegistryMap()); }
@Test public void testDeleteMappings() { TaskAssignmentManager taskAssignmentManager = new TaskAssignmentManager(config, new MetricsRegistryMap()); Map<String, String> expectedMap = ImmutableMap.of("Task0", "0", "Task1", "1"); for (Map.Entry<String, String> entry : expectedMap.entrySet()) { taskAssignmentManager.writeTaskContainerMapping(entry.getKey(), entry.getValue(), TaskMode.Active); } Map<String, String> localMap = taskAssignmentManager.readTaskAssignment(); assertEquals(expectedMap, localMap); taskAssignmentManager.deleteTaskContainerMappings(localMap.keySet()); Map<String, String> deletedMap = taskAssignmentManager.readTaskAssignment(); assertTrue(deletedMap.isEmpty()); taskAssignmentManager.close(); }
/** * Build ContainerModels from job config file and put the results in the containerModels map. */ private void getContainerModels() { MetricsRegistryMap metricsRegistryMap = new MetricsRegistryMap(); CoordinatorStreamManager coordinatorStreamManager = new CoordinatorStreamManager(jobConfig, metricsRegistryMap); coordinatorStreamManager.register(getClass().getSimpleName()); coordinatorStreamManager.start(); coordinatorStreamManager.bootstrap(); ChangelogStreamManager changelogStreamManager = new ChangelogStreamManager(coordinatorStreamManager); JobModel jobModel = JobModelManager.apply(coordinatorStreamManager.getConfig(), changelogStreamManager.readPartitionMapping(), metricsRegistryMap).jobModel(); containers = jobModel.getContainers(); coordinatorStreamManager.stop(); }
@Before public void beforeTest() { String testZkConnectionString = String.format("%s:%s", LOCALHOST, zkServer.getPort()); Config zkConfig = new MapConfig(ImmutableMap.of(ZkConfig.ZK_CONNECT, testZkConnectionString)); zkMetadataStore = new ZkMetadataStoreFactory().getMetadataStore(String.format("/%s", RandomStringUtils.randomAlphabetic(5)), zkConfig, new MetricsRegistryMap()); }
@Before public void setUp() { MockCoordinatorStreamSystemFactory.enableMockConsumerCache(); Map<String, String> configMap = ImmutableMap.of("job.name", "test-job", "job.coordinator.system", "test-kafka"); MockCoordinatorStreamSystemFactory systemFactory = new MockCoordinatorStreamSystemFactory(); PowerMockito.mockStatic(CoordinatorStreamUtil.class); when(CoordinatorStreamUtil.getCoordinatorSystemFactory(anyObject())).thenReturn(systemFactory); when(CoordinatorStreamUtil.getCoordinatorSystemStream(anyObject())).thenReturn(new SystemStream("test-kafka", "test")); when(CoordinatorStreamUtil.getCoordinatorStreamName(anyObject(), anyObject())).thenReturn("test"); coordinatorStreamStore = new CoordinatorStreamStore(SetTaskContainerMapping.TYPE, new MapConfig(configMap), new MetricsRegistryMap()); coordinatorStreamStore.init(); }
@Test public void testWriteOnlyLocalityManager() { LocalityManager localityManager = new LocalityManager(config, new MetricsRegistryMap()); localityManager.writeContainerToHostMapping("1", "localhost"); assertEquals(localityManager.readContainerLocality().size(), 1); assertEquals(ImmutableMap.of("1", ImmutableMap.of("host", "localhost")), localityManager.readContainerLocality()); localityManager.close(); MockCoordinatorStreamSystemProducer producer = mockCoordinatorStreamSystemFactory.getCoordinatorStreamSystemProducer(config, null); MockCoordinatorStreamSystemConsumer consumer = mockCoordinatorStreamSystemFactory.getCoordinatorStreamSystemConsumer(config, null); assertTrue(producer.isStopped()); assertTrue(consumer.isStopped()); } }