/** * run the setup phase and restore all the task storages */ public void run() { setup(); log.info("start recovering..."); systemAdmins.start(); this.containerStorageManagers.forEach((containerName, containerStorageManager) -> { containerStorageManager.start(); }); this.containerStorageManagers.forEach((containerName, containerStorageManager) -> { containerStorageManager.shutdown(); }); systemAdmins.stop(); log.info("successfully recovered in " + storeBaseDir.toString()); }
/** * Create taskStores with the given store mode for all stores in storageEngineFactories. */ private Map<TaskName, Map<String, StorageEngine>> createTaskStores(ContainerModel containerModel, JobContext jobContext, ContainerContext containerContext, Map<String, StorageEngineFactory<Object, Object>> storageEngineFactories, Map<String, SystemStream> changelogSystemStreams, Map<String, Serde<Object>> serdes, Map<TaskName, TaskInstanceMetrics> taskInstanceMetrics, Map<TaskName, TaskInstanceCollector> taskInstanceCollectors, StorageEngineFactory.StoreMode storeMode) { Map<TaskName, Map<String, StorageEngine>> taskStores = new HashMap<>(); // iterate over each task in the containerModel, and each store in storageEngineFactories for (Map.Entry<TaskName, TaskModel> task : containerModel.getTasks().entrySet()) { TaskName taskName = task.getKey(); TaskModel taskModel = task.getValue(); if (!taskStores.containsKey(taskName)) { taskStores.put(taskName, new HashMap<>()); } for (String storeName : storageEngineFactories.keySet()) { StorageEngine storageEngine = createStore(storeName, taskName, taskModel, jobContext, containerContext, storageEngineFactories, changelogSystemStreams, serdes, taskInstanceMetrics, taskInstanceCollectors, storeMode); // add created store to map taskStores.get(taskName).put(storeName, storageEngine); LOG.info("Created store {} for task {}", storeName, taskName); } } return taskStores; }
this.taskStores = createTaskStores(containerModel, jobContext, containerContext, storageEngineFactories, changelogSystemStreams, serdes, taskInstanceMetrics, taskInstanceCollectors, StorageEngineFactory.StoreMode.BulkLoad); this.systemConsumers = createStoreConsumers(changelogSystemStreams, systemFactories, config, this.samzaContainerMetrics.registry()); this.taskRestoreManagers = createTaskRestoreManagers(systemAdmins, clock);
/** * create one TaskStorageManager for each task. Add all of them to the * List<TaskStorageManager> */ @SuppressWarnings({"unchecked", "rawtypes"}) private void getContainerStorageManagers() { Clock clock = SystemClock.instance(); StreamMetadataCache streamMetadataCache = new StreamMetadataCache(systemAdmins, 5000, clock); // don't worry about prefetching for this; looks like the tool doesn't flush to offset files anyways Map<String, SystemFactory> systemFactories = new JavaSystemConfig(jobConfig).getSystemFactories(); for (ContainerModel containerModel : containers.values()) { ContainerContext containerContext = new ContainerContextImpl(containerModel, new MetricsRegistryMap()); ContainerStorageManager containerStorageManager = new ContainerStorageManager(containerModel, streamMetadataCache, systemAdmins, changeLogSystemStreams, storageEngineFactories, systemFactories, this.getSerdes(), jobConfig, new HashMap<>(), new SamzaContainerMetrics(containerModel.getId(), new MetricsRegistryMap()), JobContextImpl.fromConfigWithDefaults(jobConfig), containerContext, new HashMap<>(), storeBaseDir, storeBaseDir, maxPartitionNumber, new SystemClock()); this.containerStorageManagers.put(containerModel.getId(), containerStorageManager); } } }
recreatePersistentTaskStoresInReadWriteMode(this.containerModel, jobContext, containerContext, storageEngineFactories, changelogSystemStreams, serdes, taskInstanceMetrics, taskInstanceCollectors);
new ContainerStorageManager(new ContainerModel("samza-container-test", tasks), mockStreamMetadataCache, mockSystemAdmins, changelogSystemStreams, storageEngineFactories, systemFactories, serdes, config, taskInstanceMetrics, samzaContainerMetrics, Mockito.mock(JobContext.class),
@Test public void testParallelismAndMetrics() { this.containerStorageManager.start(); this.containerStorageManager.shutdown(); for (Gauge gauge : taskRestoreMetricGauges.values()) { Assert.assertTrue("Restoration time gauge value should be invoked atleast once", Mockito.mockingDetails(gauge).getInvocations().size() >= 1); } Assert.assertTrue("Store restore count should be 2 because there are 2 tasks", this.storeRestoreCallCount == 2); Assert.assertTrue("systemConsumerCreation count should be 1 (1 consumer per system)", this.systemConsumerCreationCount == 1); Assert.assertTrue("systemConsumerStopCount count should be 1", this.systemConsumerStopCount == 1); Assert.assertTrue("systemConsumerStartCount count should be 1", this.systemConsumerStartCount == 1); } }
createStore(storeName, taskName, taskModel, jobContext, containerContext, storageEngineFactories, changelogSystemStreams, serdes, taskInstanceMetrics, taskInstanceCollectors, StorageEngineFactory.StoreMode.ReadWrite);