@Nullable private File getPluginArchive(ProgramOptions options) { if (!options.getArguments().hasOption(ProgramOptionConstants.PLUGIN_ARCHIVE)) { return null; } return new File(options.getArguments().getOption(ProgramOptionConstants.PLUGIN_ARCHIVE)); } }
@Nullable private File getPluginArchive(ProgramOptions options) { if (!options.getArguments().hasOption(ProgramOptionConstants.PLUGIN_ARCHIVE)) { return null; } return new File(options.getArguments().getOption(ProgramOptionConstants.PLUGIN_ARCHIVE)); } }
@Nullable private File getPluginArchive(ProgramOptions options) { if (!options.getArguments().hasOption(ProgramOptionConstants.PLUGIN_ARCHIVE)) { return null; } return new File(options.getArguments().getOption(ProgramOptionConstants.PLUGIN_ARCHIVE)); } }
@Nullable private File getPluginArchive(ProgramOptions options) { if (!options.getArguments().hasOption(ProgramOptionConstants.PLUGIN_ARCHIVE)) { return null; } return new File(options.getArguments().getOption(ProgramOptionConstants.PLUGIN_ARCHIVE)); } }
@Nullable private File getPluginArchive(ProgramOptions options) { if (!options.getArguments().hasOption(ProgramOptionConstants.PLUGIN_ARCHIVE)) { return null; } return new File(options.getArguments().getOption(ProgramOptionConstants.PLUGIN_ARCHIVE)); } }
/** * Creates a {@link PluginInstantiator} based on the {@link ProgramOptionConstants#PLUGIN_DIR} in * the system arguments in the given {@link ProgramOptions}. * * @param options the program runner options * @param classLoader the parent ClassLoader for the {@link PluginInstantiator} to use * @return A new {@link PluginInstantiator} or {@code null} if no plugin is available. */ @Nullable protected PluginInstantiator createPluginInstantiator(ProgramOptions options, ClassLoader classLoader) { if (!options.getArguments().hasOption(ProgramOptionConstants.PLUGIN_DIR)) { return null; } return new PluginInstantiator( cConf, classLoader, new File(options.getArguments().getOption(ProgramOptionConstants.PLUGIN_DIR))); }
/** * Creates a {@link PluginInstantiator} based on the {@link ProgramOptionConstants#PLUGIN_DIR} in * the system arguments in the given {@link ProgramOptions}. * * @param options the program runner options * @param classLoader the parent ClassLoader for the {@link PluginInstantiator} to use * @return A new {@link PluginInstantiator} or {@code null} if no plugin is available. */ @Nullable protected PluginInstantiator createPluginInstantiator(ProgramOptions options, ClassLoader classLoader) { if (!options.getArguments().hasOption(ProgramOptionConstants.PLUGIN_DIR)) { return null; } return new PluginInstantiator( cConf, classLoader, new File(options.getArguments().getOption(ProgramOptionConstants.PLUGIN_DIR))); }
/** * Creates a new instance of {@link ProgramOptions} with artifact localization information and with * extra system arguments, while maintaining other fields of the given {@link ProgramOptions}. * * @param options the original {@link ProgramOptions}. * @param localizeResources a {@link Map} of {@link LocalizeResource} to be localized to the remote container * @param tempDir a local temporary directory for creating files for artifact localization. * @param extraSystemArgs a set of extra system arguments to be added/updated * @return a new instance of {@link ProgramOptions} * @throws IOException if failed to create local copy of artifact files */ private ProgramOptions updateProgramOptions(ProgramOptions options, Map<String, LocalizeResource> localizeResources, File tempDir, Map<String, String> extraSystemArgs) throws IOException { Arguments systemArgs = options.getArguments(); Map<String, String> newSystemArgs = new HashMap<>(systemArgs.asMap()); newSystemArgs.putAll(extraSystemArgs); if (systemArgs.hasOption(ProgramOptionConstants.PLUGIN_DIR)) { File localDir = new File(systemArgs.getOption(ProgramOptionConstants.PLUGIN_DIR)); File archiveFile = new File(tempDir, "artifacts.jar"); BundleJarUtil.createJar(localDir, archiveFile); // Localize plugins to two files, one expanded into a directory, one not. localizeResources.put("artifacts", new LocalizeResource(archiveFile, true)); localizeResources.put("artifacts_archive.jar", new LocalizeResource(archiveFile, false)); newSystemArgs.put(ProgramOptionConstants.PLUGIN_DIR, "artifacts"); newSystemArgs.put(ProgramOptionConstants.PLUGIN_ARCHIVE, "artifacts_archive.jar"); } return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(newSystemArgs), options.getUserArguments(), options.isDebug()); }
/** * Creates a new instance of {@link ProgramOptions} with artifact localization information and with * extra system arguments, while maintaining other fields of the given {@link ProgramOptions}. * * @param options the original {@link ProgramOptions}. * @param localizeResources a {@link Map} of {@link LocalizeResource} to be localized to the remote container * @param tempDir a local temporary directory for creating files for artifact localization. * @param extraSystemArgs a set of extra system arguments to be added/updated * @return a new instance of {@link ProgramOptions} * @throws IOException if failed to create local copy of artifact files */ private ProgramOptions updateProgramOptions(ProgramOptions options, Map<String, LocalizeResource> localizeResources, File tempDir, Map<String, String> extraSystemArgs) throws IOException { Arguments systemArgs = options.getArguments(); Map<String, String> newSystemArgs = new HashMap<>(systemArgs.asMap()); newSystemArgs.putAll(extraSystemArgs); if (systemArgs.hasOption(ProgramOptionConstants.PLUGIN_DIR)) { File localDir = new File(systemArgs.getOption(ProgramOptionConstants.PLUGIN_DIR)); File archiveFile = new File(tempDir, "artifacts.jar"); BundleJarUtil.createJar(localDir, archiveFile); // Localize plugins to two files, one expanded into a directory, one not. localizeResources.put("artifacts", new LocalizeResource(archiveFile, true)); localizeResources.put("artifacts_archive.jar", new LocalizeResource(archiveFile, false)); newSystemArgs.put(ProgramOptionConstants.PLUGIN_DIR, "artifacts"); newSystemArgs.put(ProgramOptionConstants.PLUGIN_ARCHIVE, "artifacts_archive.jar"); } return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(newSystemArgs), options.getUserArguments(), options.isDebug()); }
private List<Module> getCoreModules(final ProgramId programId) { ClusterMode clusterMode = systemArgs.hasOption(ProgramOptionConstants.CLUSTER_MODE) ? ClusterMode.valueOf(systemArgs.getOption(ProgramOptionConstants.CLUSTER_MODE)) : ClusterMode.ON_PREMISE;
public DistributedSparkSubmitter(Configuration hConf, LocationFactory locationFactory, String hostname, SparkRuntimeContext runtimeContext, @Nullable String schedulerQueueName) { this.hConf = hConf; this.schedulerQueueName = schedulerQueueName; ProgramRunId programRunId = runtimeContext.getProgram().getId().run(runtimeContext.getRunId().getId()); WorkflowProgramInfo workflowInfo = runtimeContext.getWorkflowInfo(); BasicWorkflowToken workflowToken = workflowInfo == null ? null : workflowInfo.getWorkflowToken(); this.sparkExecutionService = new SparkExecutionService(locationFactory, hostname, programRunId, workflowToken); Arguments systemArgs = runtimeContext.getProgramOptions().getArguments(); this.tokenRenewalInterval = systemArgs.hasOption(SparkRuntimeContextConfig.CREDENTIALS_UPDATE_INTERVAL_MS) ? Long.parseLong(systemArgs.getOption(SparkRuntimeContextConfig.CREDENTIALS_UPDATE_INTERVAL_MS)) : -1L; }
public DistributedSparkSubmitter(Configuration hConf, LocationFactory locationFactory, String hostname, SparkRuntimeContext runtimeContext, @Nullable String schedulerQueueName) { this.hConf = hConf; this.schedulerQueueName = schedulerQueueName; ProgramRunId programRunId = runtimeContext.getProgram().getId().run(runtimeContext.getRunId().getId()); WorkflowProgramInfo workflowInfo = runtimeContext.getWorkflowInfo(); BasicWorkflowToken workflowToken = workflowInfo == null ? null : workflowInfo.getWorkflowToken(); this.sparkExecutionService = new SparkExecutionService(locationFactory, hostname, programRunId, workflowToken); Arguments systemArgs = runtimeContext.getProgramOptions().getArguments(); this.tokenRenewalInterval = systemArgs.hasOption(SparkRuntimeContextConfig.CREDENTIALS_UPDATE_INTERVAL_MS) ? Long.parseLong(systemArgs.getOption(SparkRuntimeContextConfig.CREDENTIALS_UPDATE_INTERVAL_MS)) : -1L; }
public DistributedSparkSubmitter(Configuration hConf, LocationFactory locationFactory, String hostname, SparkRuntimeContext runtimeContext, @Nullable String schedulerQueueName) { this.hConf = hConf; this.schedulerQueueName = schedulerQueueName; ProgramRunId programRunId = runtimeContext.getProgram().getId().run(runtimeContext.getRunId().getId()); WorkflowProgramInfo workflowInfo = runtimeContext.getWorkflowInfo(); BasicWorkflowToken workflowToken = workflowInfo == null ? null : workflowInfo.getWorkflowToken(); this.sparkExecutionService = new SparkExecutionService(locationFactory, hostname, programRunId, workflowToken); Arguments systemArgs = runtimeContext.getProgramOptions().getArguments(); this.tokenRenewalInterval = systemArgs.hasOption(SparkRuntimeContextConfig.CREDENTIALS_UPDATE_INTERVAL_MS) ? Long.parseLong(systemArgs.getOption(SparkRuntimeContextConfig.CREDENTIALS_UPDATE_INTERVAL_MS)) : -1L; }
private List<Module> getCoreModules(final ProgramId programId) { ClusterMode clusterMode = systemArgs.hasOption(ProgramOptionConstants.CLUSTER_MODE) ? ClusterMode.valueOf(systemArgs.getOption(ProgramOptionConstants.CLUSTER_MODE)) : ClusterMode.ON_PREMISE;