private Map<TaskName, TaskRestoreManager> createTaskRestoreManagers(SystemAdmins systemAdmins, Clock clock) { Map<TaskName, TaskRestoreManager> taskRestoreManagers = new HashMap<>(); containerModel.getTasks().forEach((taskName, taskModel) -> taskRestoreManagers.put(taskName, new TaskRestoreManager(taskModel, changelogSystemStreams, taskStores.get(taskName), systemAdmins, clock))); return taskRestoreManagers; }
/** * Determines the starting offset for each store SSP (based on {@link #getStartingOffset(SystemStreamPartition, SystemAdmin)}) and * registers it with the respective SystemConsumer for starting consumption. */ private void registerStartingOffsets() { for (Map.Entry<String, SystemStream> changelogSystemStreamEntry : changelogSystemStreams.entrySet()) { SystemStreamPartition systemStreamPartition = new SystemStreamPartition(changelogSystemStreamEntry.getValue(), taskModel.getChangelogPartition()); SystemAdmin systemAdmin = systemAdmins.getSystemAdmin(changelogSystemStreamEntry.getValue().getSystem()); SystemConsumer systemConsumer = systemConsumers.get(changelogSystemStreamEntry.getKey()); String offset = getStartingOffset(systemStreamPartition, systemAdmin); if (offset != null) { LOG.info("Registering change log consumer with offset " + offset + " for %" + systemStreamPartition); systemConsumer.register(systemStreamPartition, offset); } else { LOG.info("Skipping change log restoration for {} because stream appears to be empty (offset was null).", systemStreamPartition); taskStoresToRestore.remove(changelogSystemStreamEntry.getKey()); } } }
this.taskRestoreManagers.values().forEach(taskStorageManager -> taskStorageManager.initialize());
if (!isLoggedStoreValid(storeName, loggedStorePartitionDir)) { LOG.info("Deleting logged storage partition directory " + loggedStorePartitionDir.toPath().toString()); FileUtil.rm(loggedStorePartitionDir);
@Override public Void call() { long startTime = System.currentTimeMillis(); LOG.info("Starting stores in task instance {}", this.taskName.getTaskName()); taskRestoreManager.restoreStores(); // Stop all persistent stores after restoring. Certain persistent stores opened in BulkLoad mode are compacted // on stop, so paralleling stop() also parallelizes their compaction (a time-intensive operation). taskRestoreManager.stopPersistentStores(); long timeToRestore = System.currentTimeMillis() - startTime; if (this.samzaContainerMetrics != null) { Gauge taskGauge = this.samzaContainerMetrics.taskStoreRestorationMetrics().getOrDefault(this.taskName, null); if (taskGauge != null) { taskGauge.set(timeToRestore); } } return null; } }
/** * Get the oldest offset for each changelog SSP based on the stream's metadata (obtained from streamMetadataCache). */ private void getOldestChangeLogOffsets() { Map<SystemStream, SystemStreamMetadata> changeLogMetadata = JavaConverters.mapAsJavaMapConverter( streamMetadataCache.getStreamMetadata( JavaConverters.asScalaSetConverter(new HashSet<>(changelogSystemStreams.values())).asScala().toSet(), false)).asJava(); LOG.info("Got change log stream metadata: {}", changeLogMetadata); changeLogOldestOffsets = getChangeLogOldestOffsetsForPartition(taskModel.getChangelogPartition(), changeLogMetadata); LOG.info("Assigning oldest change log offsets for taskName {} : {}", taskModel.getTaskName(), changeLogOldestOffsets); }
/** * Cleans up and sets up store directories, validates changeLog SSPs for all stores of this task, * and registers SSPs with the respective consumers. */ public void initialize() { cleanBaseDirsAndReadOffsetFiles(); setupBaseDirs(); validateChangelogStreams(); getOldestChangeLogOffsets(); registerStartingOffsets(); }