ExecutionPlan getExecutionPlan(String runId) { Map<String, String> allowedUserConfig = new HashMap<>(userConfig); Map<String, String> generatedConfig = new HashMap<>(); // TODO: This should all be consolidated with ExecutionPlanner after fixing SAMZA-1811 // Don't generate any configurations for LegacyTaskApplications if (!LegacyTaskApplication.class.isAssignableFrom(appDesc.getAppClass())) { if (userConfig.containsKey(TaskConfig.INPUT_STREAMS())) { LOG.warn("SamzaApplications should not specify task.inputs in configuration. " + "Specify them using InputDescriptors instead. Ignoring configured task.inputs value of " + userConfig.get(TaskConfig.INPUT_STREAMS())); allowedUserConfig.remove(TaskConfig.INPUT_STREAMS()); } generatedConfig.putAll(getGeneratedConfig(runId)); } // merge user-provided configuration with generated configuration. generated configuration has lower priority. Config mergedConfig = JobNodeConfigurationGenerator.mergeConfig(allowedUserConfig, generatedConfig); // creating the StreamManager to get all input/output streams' metadata for planning StreamManager streamManager = buildAndStartStreamManager(mergedConfig); try { ExecutionPlanner planner = new ExecutionPlanner(mergedConfig, streamManager); return planner.plan(appDesc); } finally { streamManager.stop(); } }
ExecutionPlan getExecutionPlan(String runId) { Map<String, String> allowedUserConfig = new HashMap<>(userConfig); Map<String, String> generatedConfig = new HashMap<>(); // TODO: This should all be consolidated with ExecutionPlanner after fixing SAMZA-1811 // Don't generate any configurations for LegacyTaskApplications if (!LegacyTaskApplication.class.isAssignableFrom(appDesc.getAppClass())) { if (userConfig.containsKey(TaskConfig.INPUT_STREAMS())) { LOG.warn("SamzaApplications should not specify task.inputs in configuration. " + "Specify them using InputDescriptors instead. Ignoring configured task.inputs value of " + userConfig.get(TaskConfig.INPUT_STREAMS())); allowedUserConfig.remove(TaskConfig.INPUT_STREAMS()); } generatedConfig.putAll(getGeneratedConfig(runId)); } // merge user-provided configuration with generated configuration. generated configuration has lower priority. Config mergedConfig = JobNodeConfigurationGenerator.mergeConfig(allowedUserConfig, generatedConfig); // creating the StreamManager to get all input/output streams' metadata for planning StreamManager streamManager = buildAndStartStreamManager(mergedConfig); try { ExecutionPlanner planner = new ExecutionPlanner(mergedConfig, streamManager); return planner.plan(appDesc); } finally { streamManager.stop(); } }
ExecutionPlan getExecutionPlan(String runId) { Map<String, String> allowedUserConfig = new HashMap<>(userConfig); Map<String, String> generatedConfig = new HashMap<>(); // TODO: This should all be consolidated with ExecutionPlanner after fixing SAMZA-1811 // Don't generate any configurations for LegacyTaskApplications if (!LegacyTaskApplication.class.isAssignableFrom(appDesc.getAppClass())) { if (userConfig.containsKey(TaskConfig.INPUT_STREAMS())) { LOG.warn("SamzaApplications should not specify task.inputs in configuration. " + "Specify them using InputDescriptors instead. Ignoring configured task.inputs value of " + userConfig.get(TaskConfig.INPUT_STREAMS())); allowedUserConfig.remove(TaskConfig.INPUT_STREAMS()); } generatedConfig.putAll(getGeneratedConfig(runId)); } // merge user-provided configuration with generated configuration. generated configuration has lower priority. Config mergedConfig = JobNodeConfigurationGenerator.mergeConfig(allowedUserConfig, generatedConfig); // creating the StreamManager to get all input/output streams' metadata for planning StreamManager streamManager = buildAndStartStreamManager(mergedConfig); try { ExecutionPlanner planner = new ExecutionPlanner(mergedConfig, streamManager); return planner.plan(appDesc); } finally { streamManager.stop(); } }
ExecutionPlan getExecutionPlan(String runId) { Map<String, String> allowedUserConfig = new HashMap<>(userConfig); Map<String, String> generatedConfig = new HashMap<>(); // TODO: This should all be consolidated with ExecutionPlanner after fixing SAMZA-1811 // Don't generate any configurations for LegacyTaskApplications if (!LegacyTaskApplication.class.isAssignableFrom(appDesc.getAppClass())) { if (userConfig.containsKey(TaskConfig.INPUT_STREAMS())) { LOG.warn("SamzaApplications should not specify task.inputs in configuration. " + "Specify them using InputDescriptors instead. Ignoring configured task.inputs value of " + userConfig.get(TaskConfig.INPUT_STREAMS())); allowedUserConfig.remove(TaskConfig.INPUT_STREAMS()); } generatedConfig.putAll(getGeneratedConfig(runId)); } // merge user-provided configuration with generated configuration. generated configuration has lower priority. Config mergedConfig = JobNodeConfigurationGenerator.mergeConfig(allowedUserConfig, generatedConfig); // creating the StreamManager to get all input/output streams' metadata for planning StreamManager streamManager = buildAndStartStreamManager(mergedConfig); try { ExecutionPlanner planner = new ExecutionPlanner(mergedConfig, streamManager); return planner.plan(appDesc); } finally { streamManager.stop(); } }
ExecutionPlan getExecutionPlan(String runId) { Map<String, String> allowedUserConfig = new HashMap<>(userConfig); Map<String, String> generatedConfig = new HashMap<>(); // TODO: This should all be consolidated with ExecutionPlanner after fixing SAMZA-1811 // Don't generate any configurations for LegacyTaskApplications if (!LegacyTaskApplication.class.isAssignableFrom(appDesc.getAppClass())) { if (userConfig.containsKey(TaskConfig.INPUT_STREAMS())) { LOG.warn("SamzaApplications should not specify task.inputs in configuration. " + "Specify them using InputDescriptors instead. Ignoring configured task.inputs value of " + userConfig.get(TaskConfig.INPUT_STREAMS())); allowedUserConfig.remove(TaskConfig.INPUT_STREAMS()); } generatedConfig.putAll(getGeneratedConfig(runId)); } // merge user-provided configuration with generated configuration. generated configuration has lower priority. Config mergedConfig = JobNodeConfigurationGenerator.mergeConfig(allowedUserConfig, generatedConfig); // creating the StreamManager to get all input/output streams' metadata for planning StreamManager streamManager = buildAndStartStreamManager(mergedConfig); try { ExecutionPlanner planner = new ExecutionPlanner(mergedConfig, streamManager); return planner.plan(appDesc); } finally { streamManager.stop(); } }
private Config getExpectedJobConfig(Config originConfig, Map<String, StreamEdge> inputEdges) { Map<String, String> configMap = new HashMap<>(originConfig); Set<String> inputs = new HashSet<>(); Set<String> broadcasts = new HashSet<>(); for (StreamEdge inputEdge : inputEdges.values()) { if (inputEdge.isBroadcast()) { broadcasts.add(inputEdge.getName() + "#0"); } else { inputs.add(inputEdge.getName()); } } if (!inputs.isEmpty()) { configMap.put(TaskConfig.INPUT_STREAMS(), Joiner.on(',').join(inputs)); } if (!broadcasts.isEmpty()) { configMap.put(TaskConfigJava.BROADCAST_INPUT_STREAMS, Joiner.on(',').join(broadcasts)); } return new MapConfig(configMap); }
generatedConfig.put(TaskConfig.INPUT_STREAMS(), Joiner.on(',').join(inputs));
generatedConfig.put(TaskConfig.INPUT_STREAMS(), Joiner.on(',').join(inputs));
generatedConfig.put(TaskConfig.INPUT_STREAMS(), Joiner.on(',').join(inputs));
generatedConfig.put(TaskConfig.INPUT_STREAMS(), Joiner.on(',').join(inputs));
generatedConfig.put(TaskConfig.INPUT_STREAMS(), Joiner.on(',').join(inputs));
private void validateJobConfig(Config expectedConfig, JobConfig jobConfig) { assertEquals(expectedConfig.get(JobConfig.JOB_NAME()), jobConfig.getName().get()); assertEquals(expectedConfig.get(JobConfig.JOB_ID()), jobConfig.getJobId()); assertEquals("testJobGraphJson", jobConfig.get(JobNodeConfigurationGenerator.CONFIG_INTERNAL_EXECUTION_PLAN)); assertEquals(expectedConfig.get(TaskConfig.INPUT_STREAMS()), jobConfig.get(TaskConfig.INPUT_STREAMS())); assertEquals(expectedConfig.get(TaskConfigJava.BROADCAST_INPUT_STREAMS), jobConfig.get(TaskConfigJava.BROADCAST_INPUT_STREAMS)); }