private void addOnPremiseServices(Injector injector, ProgramOptions programOptions, MetricsCollectionService metricsCollectionService, Collection<Service> services) { services.add(injector.getInstance(KafkaClientService.class)); services.add(injector.getInstance(BrokerService.class)); services.add(new ProgramRunnableResourceReporter(programOptions.getProgramId(), metricsCollectionService, context)); }
private void addOnPremiseServices(Injector injector, ProgramOptions programOptions, MetricsCollectionService metricsCollectionService, Collection<Service> services) { services.add(injector.getInstance(KafkaClientService.class)); services.add(injector.getInstance(BrokerService.class)); services.add(injector.getInstance(StreamCoordinatorClient.class)); services.add(new ProgramRunnableResourceReporter(programOptions.getProgramId(), metricsCollectionService, context)); }
private ProgramOptions createComponentOptions(int instanceId, int instances, RunId runId, ProgramOptions options) { Map<String, String> systemOptions = Maps.newHashMap(); systemOptions.putAll(options.getArguments().asMap()); systemOptions.put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(instanceId)); systemOptions.put(ProgramOptionConstants.INSTANCES, Integer.toString(instances)); systemOptions.put(ProgramOptionConstants.RUN_ID, runId.getId()); systemOptions.put(ProgramOptionConstants.HOST, host); return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemOptions), options.getUserArguments()); }
private ProgramOptions createComponentOptions(int instanceId, int instances, RunId runId, ProgramOptions options) { Map<String, String> systemOptions = Maps.newHashMap(); systemOptions.putAll(options.getArguments().asMap()); systemOptions.put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(instanceId)); systemOptions.put(ProgramOptionConstants.INSTANCES, Integer.toString(instances)); systemOptions.put(ProgramOptionConstants.RUN_ID, runId.getId()); systemOptions.put(ProgramOptionConstants.HOST, host); return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemOptions), options.getUserArguments()); }
private ProgramOptions createFlowletOptions(int instanceId, int instances, ProgramOptions options) { Map<String, String> systemArgs = new HashMap<>(); systemArgs.putAll(options.getArguments().asMap()); systemArgs.put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(instanceId)); systemArgs.put(ProgramOptionConstants.INSTANCES, Integer.toString(instances)); return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemArgs), options.getUserArguments()); }
@Override public JsonElement serialize(ProgramOptions src, Type typeOfSrc, JsonSerializationContext context) { JsonObject json = new JsonObject(); json.add("programId", context.serialize(src.getProgramId(), ProgramId.class)); json.add("arguments", context.serialize(src.getArguments(), Arguments.class)); json.add("userArguments", context.serialize(src.getUserArguments(), Arguments.class)); json.addProperty("debug", src.isDebug()); return json; } }
@Override public JsonElement serialize(ProgramOptions src, Type typeOfSrc, JsonSerializationContext context) { JsonObject json = new JsonObject(); json.add("programId", context.serialize(src.getProgramId(), ProgramId.class)); json.add("arguments", context.serialize(src.getArguments(), Arguments.class)); json.add("userArguments", context.serialize(src.getUserArguments(), Arguments.class)); json.addProperty("debug", src.isDebug()); return json; } }
return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(builder.build()), options.getUserArguments(), options.isDebug());
return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(builder.build()), options.getUserArguments(), options.isDebug());
/** * Creates a new instance of {@link ProgramOptions} with artifact localization information and with * extra system arguments, while maintaining other fields of the given {@link ProgramOptions}. * * @param options the original {@link ProgramOptions}. * @param localizeResources a {@link Map} of {@link LocalizeResource} to be localized to the remote container * @param tempDir a local temporary directory for creating files for artifact localization. * @param extraSystemArgs a set of extra system arguments to be added/updated * @return a new instance of {@link ProgramOptions} * @throws IOException if failed to create local copy of artifact files */ private ProgramOptions updateProgramOptions(ProgramOptions options, Map<String, LocalizeResource> localizeResources, File tempDir, Map<String, String> extraSystemArgs) throws IOException { Arguments systemArgs = options.getArguments(); Map<String, String> newSystemArgs = new HashMap<>(systemArgs.asMap()); newSystemArgs.putAll(extraSystemArgs); if (systemArgs.hasOption(ProgramOptionConstants.PLUGIN_DIR)) { File localDir = new File(systemArgs.getOption(ProgramOptionConstants.PLUGIN_DIR)); File archiveFile = new File(tempDir, "artifacts.jar"); BundleJarUtil.createJar(localDir, archiveFile); // Localize plugins to two files, one expanded into a directory, one not. localizeResources.put("artifacts", new LocalizeResource(archiveFile, true)); localizeResources.put("artifacts_archive.jar", new LocalizeResource(archiveFile, false)); newSystemArgs.put(ProgramOptionConstants.PLUGIN_DIR, "artifacts"); newSystemArgs.put(ProgramOptionConstants.PLUGIN_ARCHIVE, "artifacts_archive.jar"); } return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(newSystemArgs), options.getUserArguments(), options.isDebug()); }
/** * Creates a new instance of {@link ProgramOptions} with artifact localization information and with * extra system arguments, while maintaining other fields of the given {@link ProgramOptions}. * * @param options the original {@link ProgramOptions}. * @param localizeResources a {@link Map} of {@link LocalizeResource} to be localized to the remote container * @param tempDir a local temporary directory for creating files for artifact localization. * @param extraSystemArgs a set of extra system arguments to be added/updated * @return a new instance of {@link ProgramOptions} * @throws IOException if failed to create local copy of artifact files */ private ProgramOptions updateProgramOptions(ProgramOptions options, Map<String, LocalizeResource> localizeResources, File tempDir, Map<String, String> extraSystemArgs) throws IOException { Arguments systemArgs = options.getArguments(); Map<String, String> newSystemArgs = new HashMap<>(systemArgs.asMap()); newSystemArgs.putAll(extraSystemArgs); if (systemArgs.hasOption(ProgramOptionConstants.PLUGIN_DIR)) { File localDir = new File(systemArgs.getOption(ProgramOptionConstants.PLUGIN_DIR)); File archiveFile = new File(tempDir, "artifacts.jar"); BundleJarUtil.createJar(localDir, archiveFile); // Localize plugins to two files, one expanded into a directory, one not. localizeResources.put("artifacts", new LocalizeResource(archiveFile, true)); localizeResources.put("artifacts_archive.jar", new LocalizeResource(archiveFile, false)); newSystemArgs.put(ProgramOptionConstants.PLUGIN_DIR, "artifacts"); newSystemArgs.put(ProgramOptionConstants.PLUGIN_ARCHIVE, "artifacts_archive.jar"); } return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(newSystemArgs), options.getUserArguments(), options.isDebug()); }
/** * Creates a {@link ProgramOptions} by deserializing the given json file. */ private ProgramOptions createProgramOptions(File programOptionsFile) throws IOException { ProgramOptions original = readJsonFile(programOptionsFile, ProgramOptions.class); // Overwrite them with environmental information Map<String, String> arguments = new HashMap<>(original.getArguments().asMap()); arguments.putAll(getExtraSystemArguments()); // Use the name passed in by the constructor as the program name to construct the ProgramId return new SimpleProgramOptions(original.getProgramId(), new BasicArguments(arguments), resolveScope(original.getUserArguments()), original.isDebug()); }
/** * Creates a {@link ProgramOptions} by deserializing the given json file. */ private ProgramOptions createProgramOptions(File programOptionsFile) throws IOException { ProgramOptions original = readJsonFile(programOptionsFile, ProgramOptions.class); // Overwrite them with environmental information Map<String, String> arguments = new HashMap<>(original.getArguments().asMap()); arguments.putAll(getExtraSystemArguments()); // Use the name passed in by the constructor as the program name to construct the ProgramId return new SimpleProgramOptions(original.getProgramId(), new BasicArguments(arguments), resolveScope(original.getUserArguments()), original.isDebug()); }
private ProgramOptions resolveFlowletOptions(ProgramOptions options, String flowlet) { Map<String, String> systemArgs = new HashMap<>(options.getArguments().asMap()); systemArgs.put(ProgramOptionConstants.FLOWLET_NAME, flowlet); return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemArgs), new BasicArguments(RuntimeArguments.extractScope( FlowUtils.FLOWLET_SCOPE, flowlet, options.getUserArguments().asMap()))); }
programRunId = programOptions.getProgramId().run(ProgramRunners.getRunId(programOptions)); new File(systemArgs.getOption(ProgramOptionConstants.APP_SPEC_FILE)), ApplicationSpecification.class); program = Programs.create(cConf, programRunner, new ProgramDescriptor(programOptions.getProgramId(), appSpec), programJarLocation, new File(systemArgs.getOption(ProgramOptionConstants.EXPANDED_PROGRAM_JAR)));
userArguments); return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemArguments), new BasicArguments(userArguments), options.isDebug());
userArguments); return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemArguments), new BasicArguments(userArguments), options.isDebug());
private boolean programOptionsEquals(ProgramOptions opt1, ProgramOptions opt2) { return Objects.equals(opt1.getProgramId(), opt2.getProgramId()) && opt1.isDebug() == opt2.isDebug() && Objects.equals(opt1.getArguments().asMap(), opt2.getArguments().asMap()) && Objects.equals(opt1.getUserArguments().asMap(), opt2.getUserArguments().asMap()); } }
workflowContext.fieldLineageConsolidationEnabled()); ProgramOptions actionOptions = new SimpleProgramOptions(programOptions.getProgramId(), programOptions.getArguments(), new BasicArguments(RuntimeArguments.extractScope(
workflowContext.fieldLineageConsolidationEnabled()); ProgramOptions actionOptions = new SimpleProgramOptions(programOptions.getProgramId(), programOptions.getArguments(), new BasicArguments(RuntimeArguments.extractScope(