/** * Creates the {@link SamzaApplication} object from the task or application class name specified in {@code config} * * @param config the configuration of the application * @return the {@link SamzaApplication} object */ public static SamzaApplication fromConfig(Config config) { String appClassName = new ApplicationConfig(config).getAppClass(); if (StringUtils.isNotBlank(appClassName)) { // app.class is configured try { Class<SamzaApplication> appClass = (Class<SamzaApplication>) Class.forName(appClassName); if (StreamApplication.class.isAssignableFrom(appClass) || TaskApplication.class.isAssignableFrom(appClass)) { return appClass.newInstance(); } } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) { throw new ConfigException(String.format("Loading app.class %s failed. The user application has to implement " + "StreamApplication or TaskApplication.", appClassName), e); } } // no app.class defined. It has to be a legacy application with task.class configuration Option<String> taskClassOption = new TaskConfig(config).getTaskClass(); if (!taskClassOption.isDefined() || !StringUtils.isNotBlank(taskClassOption.getOrElse(null))) { // no task.class defined either. This is wrong. throw new ConfigException("Legacy task applications must set a non-empty task.class in configuration."); } return new LegacyTaskApplication(taskClassOption.get()); } }
ExecutionPlan getExecutionPlan(String runId) { Map<String, String> allowedUserConfig = new HashMap<>(userConfig); Map<String, String> generatedConfig = new HashMap<>(); // TODO: This should all be consolidated with ExecutionPlanner after fixing SAMZA-1811 // Don't generate any configurations for LegacyTaskApplications if (!LegacyTaskApplication.class.isAssignableFrom(appDesc.getAppClass())) { if (userConfig.containsKey(TaskConfig.INPUT_STREAMS())) { LOG.warn("SamzaApplications should not specify task.inputs in configuration. " + "Specify them using InputDescriptors instead. Ignoring configured task.inputs value of " + userConfig.get(TaskConfig.INPUT_STREAMS())); allowedUserConfig.remove(TaskConfig.INPUT_STREAMS()); } generatedConfig.putAll(getGeneratedConfig(runId)); } // merge user-provided configuration with generated configuration. generated configuration has lower priority. Config mergedConfig = JobNodeConfigurationGenerator.mergeConfig(allowedUserConfig, generatedConfig); // creating the StreamManager to get all input/output streams' metadata for planning StreamManager streamManager = buildAndStartStreamManager(mergedConfig); try { ExecutionPlanner planner = new ExecutionPlanner(mergedConfig, streamManager); return planner.plan(appDesc); } finally { streamManager.stop(); } }
private void configureWindowInterval(Map<String, String> configs, Config config, Collection<OperatorSpec> reachableOperators) { if (!reachableOperators.stream().anyMatch(op -> op.getOpCode() == OperatorSpec.OpCode.WINDOW || op.getOpCode() == OperatorSpec.OpCode.JOIN)) { return; } // set triggering interval if a window or join is defined. Only applies to high-level applications long triggerInterval = computeTriggerInterval(reachableOperators); LOG.info("Using triggering interval: {}", triggerInterval); configs.put(TaskConfig.WINDOW_MS(), String.valueOf(triggerInterval)); }
/** * Get the SystemStreams for the configured input and broadcast streams. * * @return the set of SystemStreams for both standard inputs and broadcast stream inputs. */ public Set<SystemStream> getAllInputStreams() { Set<SystemStream> allInputSS = new HashSet<>(); TaskConfig taskConfig = TaskConfig.Config2Task(this); allInputSS.addAll((Set<? extends SystemStream>) JavaConverters.setAsJavaSetConverter(taskConfig.getInputStreams()).asJava()); allInputSS.addAll(getBroadcastSystemStreams()); return Collections.unmodifiableSet(allInputSS); }
/** * For each input stream specified in config, exactly determine its * partitions, returning a set of SystemStreamPartitions containing them all. */ private Set<SystemStreamPartition> getInputStreamPartitions() { TaskConfig taskConfig = new TaskConfig(config); scala.collection.immutable.Set<SystemStream> inputSystemStreams = taskConfig.getInputStreams(); // Get the set of partitions for each SystemStream from the stream metadata Set<SystemStreamPartition> sspSet = JavaConverters.mapAsJavaMapConverter(streamMetadataCache.getStreamMetadata(inputSystemStreams, true)).asJava() .entrySet() .stream() .flatMap(this::mapSSMToSSP) .collect(Collectors.toSet()); return sspSet; }
HighResolutionClock clock) { long taskWindowMs = config.getWindowMs(); long taskCommitMs = config.getCommitMs(); toScalaFunction(() -> clock.nanoTime())); } else { Integer taskMaxConcurrency = config.getMaxConcurrency(); boolean isAsyncCommitEnabled = config.getAsyncCommit(); Long callbackTimeout = config.getCallbackTimeoutMs(); Long maxIdleMs = config.getMaxIdleMs();
TaskConfig taskConfig = new TaskConfig(prevConfig); String checkpointManagerFactoryClassName = taskConfig.getCheckpointManagerFactory() .getOrElse(defaultValue(null)); if (checkpointManagerFactoryClassName != null) {
public AbstractContainerAllocator(ClusterResourceManager containerProcessManager, ResourceRequestState resourceRequestState, Config config, SamzaApplicationState state) { ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config); this.clusterResourceManager = containerProcessManager; this.allocatorSleepIntervalMs = clusterManagerConfig.getAllocatorSleepTime(); this.resourceRequestState = resourceRequestState; this.containerMemoryMb = clusterManagerConfig.getContainerMemoryMb(); this.containerNumCpuCores = clusterManagerConfig.getNumCores(); this.taskConfig = new TaskConfig(config); this.state = state; this.config = config; }
/** * Returns a command builder with the build environment configured with the containerId. * @param samzaContainerId to configure the builder with. * @return the constructed builder object */ private CommandBuilder getCommandBuilder(String samzaContainerId) { String cmdBuilderClassName = taskConfig.getCommandClass(ShellCommandBuilder.class.getName()); CommandBuilder cmdBuilder = Util.getObj(cmdBuilderClassName, CommandBuilder.class); cmdBuilder.setConfig(config).setId(samzaContainerId).setUrl(state.jobModelManager.server().getUrl()); return cmdBuilder; } /**
@Test public void testAutoCommitConfig() { // positive values of commit.ms => autoCommit = true Config config1 = new MapConfig(ImmutableMap.of("task.commit.ms", "1")); assertTrue(new TaskConfig(config1).isAutoCommitEnabled()); // no value for commit.ms => autoCommit = true Config config2 = new MapConfig(ImmutableMap.of()); assertTrue(new TaskConfig(config2).isAutoCommitEnabled()); // A zero value for commit.ms => autoCommit = false Config config3 = new MapConfig(ImmutableMap.of("task.commit.ms", "0")); assertFalse(new TaskConfig(config3).isAutoCommitEnabled()); // negative value for commit.ms => autoCommit = false Config config4 = new MapConfig(ImmutableMap.of("task.commit.ms", "-1")); assertFalse(new TaskConfig(config4).isAutoCommitEnabled()); } }
HighResolutionClock clock) { long taskWindowMs = config.getWindowMs(); long taskCommitMs = config.getCommitMs(); toScalaFunction(() -> clock.nanoTime())); } else { Integer taskMaxConcurrency = config.getMaxConcurrency(); boolean isAsyncCommitEnabled = config.getAsyncCommit(); Long callbackTimeout = config.getCallbackTimeoutMs(); Long maxIdleMs = config.getMaxIdleMs();
TaskConfig taskConfig = new TaskConfig(prevConfig); String checkpointManagerFactoryClassName = taskConfig.getCheckpointManagerFactory() .getOrElse(defaultValue(null)); if (checkpointManagerFactoryClassName != null) {
/** * Get the SystemStreams for the configured input and broadcast streams. * * @return the set of SystemStreams for both standard inputs and broadcast stream inputs. */ public Set<SystemStream> getAllInputStreams() { Set<SystemStream> allInputSS = new HashSet<>(); TaskConfig taskConfig = TaskConfig.Config2Task(this); allInputSS.addAll((Set<? extends SystemStream>) JavaConverters.setAsJavaSetConverter(taskConfig.getInputStreams()).asJava()); allInputSS.addAll(getBroadcastSystemStreams()); return Collections.unmodifiableSet(allInputSS); }
public AbstractContainerAllocator(ClusterResourceManager containerProcessManager, ResourceRequestState resourceRequestState, Config config, SamzaApplicationState state) { ClusterManagerConfig clusterManagerConfig = new ClusterManagerConfig(config); this.clusterResourceManager = containerProcessManager; this.allocatorSleepIntervalMs = clusterManagerConfig.getAllocatorSleepTime(); this.resourceRequestState = resourceRequestState; this.containerMemoryMb = clusterManagerConfig.getContainerMemoryMb(); this.containerNumCpuCores = clusterManagerConfig.getNumCores(); this.taskConfig = new TaskConfig(config); this.state = state; this.config = config; }
/** * Returns a command builder with the build environment configured with the containerId. * @param samzaContainerId to configure the builder with. * @return the constructed builder object */ private CommandBuilder getCommandBuilder(String samzaContainerId) { String cmdBuilderClassName = taskConfig.getCommandClass(ShellCommandBuilder.class.getName()); CommandBuilder cmdBuilder = Util.getObj(cmdBuilderClassName, CommandBuilder.class); cmdBuilder.setConfig(config).setId(samzaContainerId).setUrl(state.jobModelManager.server().getUrl()); return cmdBuilder; } /**
HighResolutionClock clock) { long taskWindowMs = config.getWindowMs(); long taskCommitMs = config.getCommitMs(); toScalaFunction(() -> clock.nanoTime())); } else { Integer taskMaxConcurrency = config.getMaxConcurrency(); boolean isAsyncCommitEnabled = config.getAsyncCommit(); Long callbackTimeout = config.getCallbackTimeoutMs(); Long maxIdleMs = config.getMaxIdleMs();
/** * Creates the {@link SamzaApplication} object from the task or application class name specified in {@code config} * * @param config the configuration of the application * @return the {@link SamzaApplication} object */ public static SamzaApplication fromConfig(Config config) { String appClassName = new ApplicationConfig(config).getAppClass(); if (StringUtils.isNotBlank(appClassName)) { // app.class is configured try { Class<SamzaApplication> appClass = (Class<SamzaApplication>) Class.forName(appClassName); if (StreamApplication.class.isAssignableFrom(appClass) || TaskApplication.class.isAssignableFrom(appClass)) { return appClass.newInstance(); } } catch (IllegalAccessException | InstantiationException | ClassNotFoundException e) { throw new ConfigException(String.format("Loading app.class %s failed. The user application has to implement " + "StreamApplication or TaskApplication.", appClassName), e); } } // no app.class defined. It has to be a legacy application with task.class configuration Option<String> taskClassOption = new TaskConfig(config).getTaskClass(); if (!taskClassOption.isDefined() || !StringUtils.isNotBlank(taskClassOption.getOrElse(null))) { // no task.class defined either. This is wrong. throw new ConfigException("Legacy task applications must set a non-empty task.class in configuration."); } return new LegacyTaskApplication(taskClassOption.get()); } }
TaskConfig taskConfig = new TaskConfig(prevConfig); String checkpointManagerFactoryClassName = taskConfig.getCheckpointManagerFactory() .getOrElse(defaultValue(null)); if (checkpointManagerFactoryClassName != null) {
/** * Get the SystemStreams for the configured input and broadcast streams. * * @return the set of SystemStreams for both standard inputs and broadcast stream inputs. */ public Set<SystemStream> getAllInputStreams() { Set<SystemStream> allInputSS = new HashSet<>(); TaskConfig taskConfig = TaskConfig.Config2Task(this); allInputSS.addAll((Set<? extends SystemStream>) JavaConverters.setAsJavaSetConverter(taskConfig.getInputStreams()).asJava()); allInputSS.addAll(getBroadcastSystemStreams()); return Collections.unmodifiableSet(allInputSS); }
ExecutionPlan getExecutionPlan(String runId) { Map<String, String> allowedUserConfig = new HashMap<>(userConfig); Map<String, String> generatedConfig = new HashMap<>(); // TODO: This should all be consolidated with ExecutionPlanner after fixing SAMZA-1811 // Don't generate any configurations for LegacyTaskApplications if (!LegacyTaskApplication.class.isAssignableFrom(appDesc.getAppClass())) { if (userConfig.containsKey(TaskConfig.INPUT_STREAMS())) { LOG.warn("SamzaApplications should not specify task.inputs in configuration. " + "Specify them using InputDescriptors instead. Ignoring configured task.inputs value of " + userConfig.get(TaskConfig.INPUT_STREAMS())); allowedUserConfig.remove(TaskConfig.INPUT_STREAMS()); } generatedConfig.putAll(getGeneratedConfig(runId)); } // merge user-provided configuration with generated configuration. generated configuration has lower priority. Config mergedConfig = JobNodeConfigurationGenerator.mergeConfig(allowedUserConfig, generatedConfig); // creating the StreamManager to get all input/output streams' metadata for planning StreamManager streamManager = buildAndStartStreamManager(mergedConfig); try { ExecutionPlanner planner = new ExecutionPlanner(mergedConfig, streamManager); return planner.plan(appDesc); } finally { streamManager.stop(); } }