/** * Returns the {@link Resources} based on configurations in the given arguments. * * Same as calling {@link #getResources(Map, Resources)} with first argument from {@link Arguments#asMap()}. */ public static Resources getResources(Arguments args, @Nullable Resources defaultResources) { return getResources(args.asMap(), defaultResources); }
/** * Optionally creates a {@link WorkflowProgramInfo} from the given arguments. If the arguments don't contain * workflow information, {@code null} will be returned. */ @Nullable public static WorkflowProgramInfo create(Arguments arguments) { String workflowName = arguments.getOption(ProgramOptionConstants.WORKFLOW_NAME); String workflowNodeId = arguments.getOption(ProgramOptionConstants.WORKFLOW_NODE_ID); String workflowRunId = arguments.getOption(ProgramOptionConstants.WORKFLOW_RUN_ID); String programNameInWorkflow = arguments.getOption(ProgramOptionConstants.PROGRAM_NAME_IN_WORKFLOW); String workflowToken = arguments.getOption(ProgramOptionConstants.WORKFLOW_TOKEN); boolean consolidateFieldOperations = Boolean.parseBoolean(arguments.getOption(ProgramOptionConstants.ENABLE_FIELD_LINEAGE_CONSOLIDATION)); if (workflowName == null || workflowNodeId == null || workflowRunId == null || workflowToken == null) { return null; } return new WorkflowProgramInfo(workflowName, workflowNodeId, workflowRunId, programNameInWorkflow, GSON.fromJson(workflowToken, BasicWorkflowToken.class), consolidateFieldOperations); }
@Override protected void setupLaunchConfig(ProgramLaunchConfig launchConfig, Program program, ProgramOptions options, CConfiguration cConf, Configuration hConf, File tempDir) { ApplicationSpecification appSpec = program.getApplicationSpecification(); WorkerSpecification workerSpec = appSpec.getWorkers().get(program.getName()); String instances = options.getArguments().getOption(ProgramOptionConstants.INSTANCES, String.valueOf(workerSpec.getInstances())); launchConfig.addRunnable(workerSpec.getName(), new WorkerTwillRunnable(workerSpec.getName()), Integer.parseInt(instances), options.getUserArguments().asMap(), workerSpec.getResources()); } }
/** * Creates a new instance of {@link ProgramOptions} with artifact localization information and with * extra system arguments, while maintaining other fields of the given {@link ProgramOptions}. * * @param options the original {@link ProgramOptions}. * @param localizeResources a {@link Map} of {@link LocalizeResource} to be localized to the remote container * @param tempDir a local temporary directory for creating files for artifact localization. * @param extraSystemArgs a set of extra system arguments to be added/updated * @return a new instance of {@link ProgramOptions} * @throws IOException if failed to create local copy of artifact files */ private ProgramOptions updateProgramOptions(ProgramOptions options, Map<String, LocalizeResource> localizeResources, File tempDir, Map<String, String> extraSystemArgs) throws IOException { Arguments systemArgs = options.getArguments(); Map<String, String> newSystemArgs = new HashMap<>(systemArgs.asMap()); newSystemArgs.putAll(extraSystemArgs); if (systemArgs.hasOption(ProgramOptionConstants.PLUGIN_DIR)) { File localDir = new File(systemArgs.getOption(ProgramOptionConstants.PLUGIN_DIR)); File archiveFile = new File(tempDir, "artifacts.jar"); BundleJarUtil.createJar(localDir, archiveFile); // Localize plugins to two files, one expanded into a directory, one not. localizeResources.put("artifacts", new LocalizeResource(archiveFile, true)); localizeResources.put("artifacts_archive.jar", new LocalizeResource(archiveFile, false)); newSystemArgs.put(ProgramOptionConstants.PLUGIN_DIR, "artifacts"); newSystemArgs.put(ProgramOptionConstants.PLUGIN_ARCHIVE, "artifacts_archive.jar"); } return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(newSystemArgs), options.getUserArguments(), options.isDebug()); }
/** * Optionally creates a {@link WorkflowProgramInfo} from the given arguments. If the arguments don't contain * workflow information, {@code null} will be returned. */ @Nullable public static WorkflowProgramInfo create(Arguments arguments) { String workflowName = arguments.getOption(ProgramOptionConstants.WORKFLOW_NAME); String workflowNodeId = arguments.getOption(ProgramOptionConstants.WORKFLOW_NODE_ID); String workflowRunId = arguments.getOption(ProgramOptionConstants.WORKFLOW_RUN_ID); String programNameInWorkflow = arguments.getOption(ProgramOptionConstants.PROGRAM_NAME_IN_WORKFLOW); String workflowToken = arguments.getOption(ProgramOptionConstants.WORKFLOW_TOKEN); boolean consolidateFieldOperations = Boolean.parseBoolean(arguments.getOption(ProgramOptionConstants.ENABLE_FIELD_LINEAGE_CONSOLIDATION)); if (workflowName == null || workflowNodeId == null || workflowRunId == null || workflowToken == null) { return null; } return new WorkflowProgramInfo(workflowName, workflowNodeId, workflowRunId, programNameInWorkflow, GSON.fromJson(workflowToken, BasicWorkflowToken.class), consolidateFieldOperations); }
@Override protected void setupLaunchConfig(ProgramLaunchConfig launchConfig, Program program, ProgramOptions options, CConfiguration cConf, Configuration hConf, File tempDir) { ApplicationSpecification appSpec = program.getApplicationSpecification(); WorkerSpecification workerSpec = appSpec.getWorkers().get(program.getName()); String instances = options.getArguments().getOption(ProgramOptionConstants.INSTANCES, String.valueOf(workerSpec.getInstances())); launchConfig.addRunnable(workerSpec.getName(), new WorkerTwillRunnable(workerSpec.getName()), Integer.parseInt(instances), options.getUserArguments().asMap(), workerSpec.getResources()); } }
/** * Creates a new instance of {@link ProgramOptions} with artifact localization information and with * extra system arguments, while maintaining other fields of the given {@link ProgramOptions}. * * @param options the original {@link ProgramOptions}. * @param localizeResources a {@link Map} of {@link LocalizeResource} to be localized to the remote container * @param tempDir a local temporary directory for creating files for artifact localization. * @param extraSystemArgs a set of extra system arguments to be added/updated * @return a new instance of {@link ProgramOptions} * @throws IOException if failed to create local copy of artifact files */ private ProgramOptions updateProgramOptions(ProgramOptions options, Map<String, LocalizeResource> localizeResources, File tempDir, Map<String, String> extraSystemArgs) throws IOException { Arguments systemArgs = options.getArguments(); Map<String, String> newSystemArgs = new HashMap<>(systemArgs.asMap()); newSystemArgs.putAll(extraSystemArgs); if (systemArgs.hasOption(ProgramOptionConstants.PLUGIN_DIR)) { File localDir = new File(systemArgs.getOption(ProgramOptionConstants.PLUGIN_DIR)); File archiveFile = new File(tempDir, "artifacts.jar"); BundleJarUtil.createJar(localDir, archiveFile); // Localize plugins to two files, one expanded into a directory, one not. localizeResources.put("artifacts", new LocalizeResource(archiveFile, true)); localizeResources.put("artifacts_archive.jar", new LocalizeResource(archiveFile, false)); newSystemArgs.put(ProgramOptionConstants.PLUGIN_DIR, "artifacts"); newSystemArgs.put(ProgramOptionConstants.PLUGIN_ARCHIVE, "artifacts_archive.jar"); } return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(newSystemArgs), options.getUserArguments(), options.isDebug()); }
@Override public JsonElement serialize(Arguments src, Type typeOfSrc, JsonSerializationContext context) { return context.serialize(src.asMap(), MAP_STRING_STRING_TYPE); } }
/** * Returns {@code true} if running in local mode. */ static boolean isLocal(ProgramOptions programOptions) { return !Boolean.parseBoolean(programOptions.getArguments().getOption(DISTRIBUTED_MODE)); }
ProgramOptions options, RunId runId) { Map<String, String> systemArguments = new HashMap<>(options.getArguments().asMap()); String clusterName = options.getArguments().getOption(Constants.CLUSTER_NAME); Map<String, String> userArguments = options.getUserArguments().asMap(); if (!Strings.isNullOrEmpty(clusterName)) { userArguments = RuntimeArguments.extractScope(CLUSTER_SCOPE, clusterName, userArguments);
/** * Returns the {@link Resources} based on configurations in the given arguments. * * Same as calling {@link #getResources(Map, Resources)} with first argument from {@link Arguments#asMap()}. */ public static Resources getResources(Arguments args, @Nullable Resources defaultResources) { return getResources(args.asMap(), defaultResources); }
/** * Returns {@code true} if running in local mode. */ static boolean isLocal(ProgramOptions programOptions) { return !Boolean.parseBoolean(programOptions.getArguments().getOption(DISTRIBUTED_MODE)); }
ProgramOptions options, RunId runId) { Map<String, String> systemArguments = new HashMap<>(options.getArguments().asMap()); String clusterName = options.getArguments().getOption(Constants.CLUSTER_NAME); Map<String, String> userArguments = options.getUserArguments().asMap(); if (!Strings.isNullOrEmpty(clusterName)) { userArguments = RuntimeArguments.extractScope(CLUSTER_SCOPE, clusterName, userArguments);
@Override public JsonElement serialize(Arguments src, Type typeOfSrc, JsonSerializationContext context) { return context.serialize(src.asMap(), MAP_STRING_STRING_TYPE); } }
/** * Returns {@code true} if running in local mode. */ static boolean isLocal(ProgramOptions programOptions) { return !Boolean.parseBoolean(programOptions.getArguments().getOption(DISTRIBUTED_MODE)); }
@Override public void start(ProgramRunId programRunId, ProgramOptions programOptions, @Nullable String twillRunId, ProgramDescriptor programDescriptor) { ImmutableMap.Builder<String, String> properties = ImmutableMap.<String, String>builder() .put(ProgramOptionConstants.PROGRAM_RUN_ID, GSON.toJson(programRunId)) .put(ProgramOptionConstants.PROGRAM_STATUS, ProgramRunStatus.STARTING.name()) .put(ProgramOptionConstants.USER_OVERRIDES, GSON.toJson(programOptions.getUserArguments().asMap())) .put(ProgramOptionConstants.SYSTEM_OVERRIDES, GSON.toJson(programOptions.getArguments().asMap())) .put(ProgramOptionConstants.PROGRAM_DESCRIPTOR, GSON.toJson(programDescriptor)); if (twillRunId != null) { properties.put(ProgramOptionConstants.TWILL_RUN_ID, twillRunId); } programStatePublisher.publish(Notification.Type.PROGRAM_STATUS, properties.build()); }