/** * Builds list of {@link Task} from job model in coordinator stream. * @param consumer system consumer associated with a job's coordinator stream. * @return list of {@link Task} constructed from job model in coordinator stream. */ protected List<Task> readTasksFromCoordinatorStream(CoordinatorStreamSystemConsumer consumer) { LocalityManager localityManager = new LocalityManager(consumer.getConfig(), new MetricsRegistryMap()); Map<String, Map<String, String>> containerIdToHostMapping = localityManager.readContainerLocality(); TaskAssignmentManager taskAssignmentManager = new TaskAssignmentManager(consumer.getConfig(), new MetricsRegistryMap()); Map<String, String> taskNameToContainerIdMapping = taskAssignmentManager.readTaskAssignment(); StorageConfig storageConfig = new StorageConfig(consumer.getConfig()); List<String> storeNames = JavaConverters.seqAsJavaListConverter(storageConfig.getStoreNames()).asJava(); return taskNameToContainerIdMapping.entrySet() .stream() .map(entry -> { String hostName = containerIdToHostMapping.get(entry.getValue()).get(SetContainerHostMapping.HOST_KEY); return new Task(hostName, entry.getKey(), entry.getValue(), new ArrayList<>(), storeNames); }).collect(Collectors.toList()); } }
/** * Directory loggedStoreDir associated with the logged store storeName is determined to be valid * if all of the following conditions are true. * a) If the store has to be persisted to disk. * b) If there is a valid offset file associated with the logged store. * c) If the logged store has not gone stale. * * @return true if the logged store is valid, false otherwise. */ private boolean isLoggedStoreValid(String storeName, File loggedStoreDir) { long changeLogDeleteRetentionInMs = StorageConfig.DEFAULT_CHANGELOG_DELETE_RETENTION_MS(); if (new StorageConfig(config).getChangeLogDeleteRetentionsInMs().get(storeName).isDefined()) { changeLogDeleteRetentionInMs = (long) new StorageConfig(config).getChangeLogDeleteRetentionsInMs().get(storeName).get(); } return this.taskStores.get(storeName).getStoreProperties().isPersistedToDisk() && StorageManagerUtil.isOffsetFileValid(loggedStoreDir, OFFSET_FILE_NAME) && !StorageManagerUtil.isStaleStore( loggedStoreDir, OFFSET_FILE_NAME, changeLogDeleteRetentionInMs, clock.currentTimeMillis()); }
Map<TaskName, TaskInstanceCollector> taskInstanceCollectors, StorageEngineFactory.StoreMode storeMode) { StorageConfig storageConfig = new StorageConfig(config);
@Override public void onBecomingLeader() { LOG.info("ZkJobCoordinator::onBecomeLeader - I became the leader"); metrics.isLeader.set(true); zkUtils.subscribeToProcessorChange(new ProcessorChangeHandler(zkUtils)); if (!new StorageConfig(config).hasDurableStores()) { // 1. Stop if there's a existing StreamPartitionCountMonitor running. if (streamPartitionCountMonitor != null) { streamPartitionCountMonitor.stop(); } // 2. Start a new instance of StreamPartitionCountMonitor. streamPartitionCountMonitor = getPartitionCountMonitor(); streamPartitionCountMonitor.start(); } debounceTimer.scheduleAfterDebounceTime(ON_PROCESSOR_CHANGE, debounceTimeMs, ZkJobCoordinator.this::doOnProcessorChange); } }
taskModel.getChangelogPartition(), systemAdmins, new StorageConfig(jobConfig).getChangeLogDeleteRetentionsInMs(), new SystemClock());
taskModel.getChangelogPartition(), systemAdmins, new StorageConfig(jobConfig).getChangeLogDeleteRetentionsInMs(), new SystemClock());
taskModel.getChangelogPartition(), systemAdmins, new StorageConfig(jobConfig).getChangeLogDeleteRetentionsInMs(), new SystemClock());
taskModel.getChangelogPartition(), systemAdmins, new StorageConfig(jobConfig).getChangeLogDeleteRetentionsInMs(), new SystemClock());
StorageConfig storageConfig = new StorageConfig(prevConfig); for (String store : JavaConversions.asJavaCollection(storageConfig.getStoreNames())) { String changelog = storageConfig.getChangelogStream(store)
StorageConfig storageConfig = new StorageConfig(prevConfig); for (String store : JavaConversions.asJavaCollection(storageConfig.getStoreNames())) { String changelog = storageConfig.getChangelogStream(store)
StorageConfig storageConfig = new StorageConfig(prevConfig); for (String store : JavaConversions.asJavaCollection(storageConfig.getStoreNames())) { String changelog = storageConfig.getChangelogStream(store)
StorageConfig storageConfig = new StorageConfig(prevConfig); for (String store : JavaConversions.asJavaCollection(storageConfig.getStoreNames())) { String changelog = storageConfig.getChangelogStream(store)
StorageConfig storageConfig = new StorageConfig(prevConfig); for (String store : JavaConversions.asJavaCollection(storageConfig.getStoreNames())) { String changelog = storageConfig.getChangelogStream(store)
hasDurableStores = new StorageConfig(config).hasDurableStores(); state = new SamzaApplicationState(jobModelManager);
hasDurableStores = new StorageConfig(config).hasDurableStores(); state = new SamzaApplicationState(jobModelManager);
hasDurableStores = new StorageConfig(config).hasDurableStores(); state = new SamzaApplicationState(jobModelManager);
hasDurableStores = new StorageConfig(config).hasDurableStores(); state = new SamzaApplicationState(jobModelManager);
hasDurableStores = new StorageConfig(config).hasDurableStores(); state = new SamzaApplicationState(jobModelManager);