/** * For each input stream specified in config, exactly determine its * partitions, returning a set of SystemStreamPartitions containing them all. */ private Set<SystemStreamPartition> getInputStreamPartitions() { TaskConfig taskConfig = new TaskConfig(config); scala.collection.immutable.Set<SystemStream> inputSystemStreams = taskConfig.getInputStreams(); // Get the set of partitions for each SystemStream from the stream metadata Set<SystemStreamPartition> sspSet = JavaConverters.mapAsJavaMapConverter(streamMetadataCache.getStreamMetadata(inputSystemStreams, true)).asJava() .entrySet() .stream() .flatMap(this::mapSSMToSSP) .collect(Collectors.toSet()); return sspSet; }
/** * Creates the {@link SamzaApplication} object from the task or application class name specified in {@code config} * * @param config the configuration of the application * @return the {@link SamzaApplication} object */ public static SamzaApplication fromConfig(Config config) { String appClassName = new ApplicationConfig(config).getAppClass(); if (StringUtils.isNotBlank(appClassName)) { // app.class is configured try { Class<SamzaApplication> appClass = (Class<SamzaApplication>) Class.forName(appClassName); if (StreamApplication.class.isAssignableFrom(appClass) || TaskApplication.class.isAssignableFrom(appClass)) { return appClass.newInstance(); } } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) { throw new ConfigException(String.format("Loading app.class %s failed. The user application has to implement " + "StreamApplication or TaskApplication.", appClassName), e); } } // no app.class defined. It has to be a legacy application with task.class configuration Option<String> taskClassOption = new TaskConfig(config).getTaskClass(); if (!taskClassOption.isDefined() || !StringUtils.isNotBlank(taskClassOption.getOrElse(null))) { // no task.class defined either. This is wrong. throw new ConfigException("Legacy task applications must set a non-empty task.class in configuration."); } return new LegacyTaskApplication(taskClassOption.get()); } }
/** * Creates the {@link SamzaApplication} object from the task or application class name specified in {@code config} * * @param config the configuration of the application * @return the {@link SamzaApplication} object */ public static SamzaApplication fromConfig(Config config) { String appClassName = new ApplicationConfig(config).getAppClass(); if (StringUtils.isNotBlank(appClassName)) { // app.class is configured try { Class<SamzaApplication> appClass = (Class<SamzaApplication>) Class.forName(appClassName); if (StreamApplication.class.isAssignableFrom(appClass) || TaskApplication.class.isAssignableFrom(appClass)) { return appClass.newInstance(); } } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) { throw new ConfigException(String.format("Loading app.class %s failed. The user application has to implement " + "StreamApplication or TaskApplication.", appClassName), e); } } // no app.class defined. It has to be a legacy application with task.class configuration Option<String> taskClassOption = new TaskConfig(config).getTaskClass(); if (!taskClassOption.isDefined() || !StringUtils.isNotBlank(taskClassOption.getOrElse(null))) { // no task.class defined either. This is wrong. throw new ConfigException("Legacy task applications must set a non-empty task.class in configuration."); } return new LegacyTaskApplication(taskClassOption.get()); } }
/** * Creates the {@link SamzaApplication} object from the task or application class name specified in {@code config} * * @param config the configuration of the application * @return the {@link SamzaApplication} object */ public static SamzaApplication fromConfig(Config config) { String appClassName = new ApplicationConfig(config).getAppClass(); if (StringUtils.isNotBlank(appClassName)) { // app.class is configured try { Class<SamzaApplication> appClass = (Class<SamzaApplication>) Class.forName(appClassName); if (StreamApplication.class.isAssignableFrom(appClass) || TaskApplication.class.isAssignableFrom(appClass)) { return appClass.newInstance(); } } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) { throw new ConfigException(String.format("Loading app.class %s failed. The user application has to implement " + "StreamApplication or TaskApplication.", appClassName), e); } } // no app.class defined. It has to be a legacy application with task.class configuration Option<String> taskClassOption = new TaskConfig(config).getTaskClass(); if (!taskClassOption.isDefined() || !StringUtils.isNotBlank(taskClassOption.getOrElse(null))) { // no task.class defined either. This is wrong. throw new ConfigException("Legacy task applications must set a non-empty task.class in configuration."); } return new LegacyTaskApplication(taskClassOption.get()); } }
/** * Creates the {@link SamzaApplication} object from the task or application class name specified in {@code config} * * @param config the configuration of the application * @return the {@link SamzaApplication} object */ public static SamzaApplication fromConfig(Config config) { String appClassName = new ApplicationConfig(config).getAppClass(); if (StringUtils.isNotBlank(appClassName)) { // app.class is configured try { Class<SamzaApplication> appClass = (Class<SamzaApplication>) Class.forName(appClassName); if (StreamApplication.class.isAssignableFrom(appClass) || TaskApplication.class.isAssignableFrom(appClass)) { return appClass.newInstance(); } } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) { throw new ConfigException(String.format("Loading app.class %s failed. The user application has to implement " + "StreamApplication or TaskApplication.", appClassName), e); } } // no app.class defined. It has to be a legacy application with task.class configuration Option<String> taskClassOption = new TaskConfig(config).getTaskClass(); if (!taskClassOption.isDefined() || !StringUtils.isNotBlank(taskClassOption.getOrElse(null))) { // no task.class defined either. This is wrong. throw new ConfigException("Legacy task applications must set a non-empty task.class in configuration."); } return new LegacyTaskApplication(taskClassOption.get()); } }
/** * Creates the {@link SamzaApplication} object from the task or application class name specified in {@code config} * * @param config the configuration of the application * @return the {@link SamzaApplication} object */ public static SamzaApplication fromConfig(Config config) { String appClassName = new ApplicationConfig(config).getAppClass(); if (StringUtils.isNotBlank(appClassName)) { // app.class is configured try { Class<SamzaApplication> appClass = (Class<SamzaApplication>) Class.forName(appClassName); if (StreamApplication.class.isAssignableFrom(appClass) || TaskApplication.class.isAssignableFrom(appClass)) { return appClass.newInstance(); } } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) { throw new ConfigException(String.format("Loading app.class %s failed. The user application has to implement " + "StreamApplication or TaskApplication.", appClassName), e); } } // no app.class defined. It has to be a legacy application with task.class configuration Option<String> taskClassOption = new TaskConfig(config).getTaskClass(); if (!taskClassOption.isDefined() || !StringUtils.isNotBlank(taskClassOption.getOrElse(null))) { // no task.class defined either. This is wrong. throw new ConfigException("Legacy task applications must set a non-empty task.class in configuration."); } return new LegacyTaskApplication(taskClassOption.get()); } }
public AbstractContainerAllocator(ClusterResourceManager containerProcessManager, ResourceRequestState resourceRequestState, Config config, SamzaApplicationState state) { ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config); this.clusterResourceManager = containerProcessManager; this.allocatorSleepIntervalMs = clusterManagerConfig.getAllocatorSleepTime(); this.resourceRequestState = resourceRequestState; this.containerMemoryMb = clusterManagerConfig.getContainerMemoryMb(); this.containerNumCpuCores = clusterManagerConfig.getNumCores(); this.taskConfig = new TaskConfig(config); this.state = state; this.config = config; }
public AbstractContainerAllocator(ClusterResourceManager containerProcessManager, ResourceRequestState resourceRequestState, Config config, SamzaApplicationState state) { ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config); this.clusterResourceManager = containerProcessManager; this.allocatorSleepIntervalMs = clusterManagerConfig.getAllocatorSleepTime(); this.resourceRequestState = resourceRequestState; this.containerMemoryMb = clusterManagerConfig.getContainerMemoryMb(); this.containerNumCpuCores = clusterManagerConfig.getNumCores(); this.taskConfig = new TaskConfig(config); this.state = state; this.config = config; }
public AbstractContainerAllocator(ClusterResourceManager containerProcessManager, ResourceRequestState resourceRequestState, Config config, SamzaApplicationState state) { ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config); this.clusterResourceManager = containerProcessManager; this.allocatorSleepIntervalMs = clusterManagerConfig.getAllocatorSleepTime(); this.resourceRequestState = resourceRequestState; this.containerMemoryMb = clusterManagerConfig.getContainerMemoryMb(); this.containerNumCpuCores = clusterManagerConfig.getNumCores(); this.taskConfig = new TaskConfig(config); this.state = state; this.config = config; }
public AbstractContainerAllocator(ClusterResourceManager containerProcessManager, ResourceRequestState resourceRequestState, Config config, SamzaApplicationState state) { ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config); this.clusterResourceManager = containerProcessManager; this.allocatorSleepIntervalMs = clusterManagerConfig.getAllocatorSleepTime(); this.resourceRequestState = resourceRequestState; this.containerMemoryMb = clusterManagerConfig.getContainerMemoryMb(); this.containerNumCpuCores = clusterManagerConfig.getNumCores(); this.taskConfig = new TaskConfig(config); this.state = state; this.config = config; }
public AbstractContainerAllocator(ClusterResourceManager containerProcessManager, ResourceRequestState resourceRequestState, Config config, SamzaApplicationState state) { ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config); this.clusterResourceManager = containerProcessManager; this.allocatorSleepIntervalMs = clusterManagerConfig.getAllocatorSleepTime(); this.resourceRequestState = resourceRequestState; this.containerMemoryMb = clusterManagerConfig.getContainerMemoryMb(); this.containerNumCpuCores = clusterManagerConfig.getNumCores(); this.taskConfig = new TaskConfig(config); this.state = state; this.config = config; }
@Test public void testAutoCommitConfig() { // positive values of commit.ms => autoCommit = true Config config1 = new MapConfig(ImmutableMap.of("task.commit.ms", "1")); assertTrue(new TaskConfig(config1).isAutoCommitEnabled()); // no value for commit.ms => autoCommit = true Config config2 = new MapConfig(ImmutableMap.of()); assertTrue(new TaskConfig(config2).isAutoCommitEnabled()); // A zero value for commit.ms => autoCommit = false Config config3 = new MapConfig(ImmutableMap.of("task.commit.ms", "0")); assertFalse(new TaskConfig(config3).isAutoCommitEnabled()); // negative value for commit.ms => autoCommit = false Config config4 = new MapConfig(ImmutableMap.of("task.commit.ms", "-1")); assertFalse(new TaskConfig(config4).isAutoCommitEnabled()); } }
TaskConfig taskConfig = new TaskConfig(prevConfig); String checkpointManagerFactoryClassName = taskConfig.getCheckpointManagerFactory() .getOrElse(defaultValue(null));
TaskConfig taskConfig = new TaskConfig(prevConfig); String checkpointManagerFactoryClassName = taskConfig.getCheckpointManagerFactory() .getOrElse(defaultValue(null));
TaskConfig taskConfig = new TaskConfig(prevConfig); String checkpointManagerFactoryClassName = taskConfig.getCheckpointManagerFactory() .getOrElse(defaultValue(null));
TaskConfig taskConfig = new TaskConfig(prevConfig); String checkpointManagerFactoryClassName = taskConfig.getCheckpointManagerFactory() .getOrElse(defaultValue(null));
TaskConfig taskConfig = new TaskConfig(prevConfig); String checkpointManagerFactoryClassName = taskConfig.getCheckpointManagerFactory() .getOrElse(defaultValue(null));