@Override public JsonElement serialize(ProgramOptions src, Type typeOfSrc, JsonSerializationContext context) { JsonObject json = new JsonObject(); json.add("programId", context.serialize(src.getProgramId(), ProgramId.class)); json.add("arguments", context.serialize(src.getArguments(), Arguments.class)); json.add("userArguments", context.serialize(src.getUserArguments(), Arguments.class)); json.addProperty("debug", src.isDebug()); return json; } }
private ProgramOptions createFlowletOptions(int instanceId, int instances, ProgramOptions options) { Map<String, String> systemArgs = new HashMap<>(); systemArgs.putAll(options.getArguments().asMap()); systemArgs.put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(instanceId)); systemArgs.put(ProgramOptionConstants.INSTANCES, Integer.toString(instances)); return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemArgs), options.getUserArguments()); }
public void provisioned(ProgramRunId programRunId, ProgramOptions programOptions, ProgramDescriptor programDescriptor, String userId, Cluster cluster, URI secureKeysDir) { Map<String, String> properties = ImmutableMap.<String, String>builder() .put(ProgramOptionConstants.PROGRAM_RUN_ID, GSON.toJson(programRunId)) .put(ProgramOptionConstants.PROGRAM_DESCRIPTOR, GSON.toJson(programDescriptor)) .put(ProgramOptionConstants.USER_ID, userId) .put(ProgramOptionConstants.CLUSTER_STATUS, ProgramRunClusterStatus.PROVISIONED.name()) .put(ProgramOptionConstants.CLUSTER, GSON.toJson(cluster)) .put(ProgramOptionConstants.DEBUG_ENABLED, String.valueOf(programOptions.isDebug())) .put(ProgramOptionConstants.USER_OVERRIDES, GSON.toJson(programOptions.getUserArguments().asMap())) .put(ProgramOptionConstants.SYSTEM_OVERRIDES, GSON.toJson(programOptions.getArguments().asMap())) .put(ProgramOptionConstants.SECURE_KEYS_DIR, GSON.toJson(secureKeysDir)) .build(); publish(properties); }
@Override public void start(ProgramRunId programRunId, ProgramOptions programOptions, @Nullable String twillRunId, ProgramDescriptor programDescriptor) { ImmutableMap.Builder<String, String> properties = ImmutableMap.<String, String>builder() .put(ProgramOptionConstants.PROGRAM_RUN_ID, GSON.toJson(programRunId)) .put(ProgramOptionConstants.PROGRAM_STATUS, ProgramRunStatus.STARTING.name()) .put(ProgramOptionConstants.USER_OVERRIDES, GSON.toJson(programOptions.getUserArguments().asMap())) .put(ProgramOptionConstants.SYSTEM_OVERRIDES, GSON.toJson(programOptions.getArguments().asMap())) .put(ProgramOptionConstants.PROGRAM_DESCRIPTOR, GSON.toJson(programDescriptor)); if (twillRunId != null) { properties.put(ProgramOptionConstants.TWILL_RUN_ID, twillRunId); } programStatePublisher.publish(Notification.Type.PROGRAM_STATUS, properties.build()); }
/** * Returns {@code true} if running in local mode. */ static boolean isLocal(ProgramOptions programOptions) { return !Boolean.parseBoolean(programOptions.getArguments().getOption(DISTRIBUTED_MODE)); }
/** * Validates the options for the program. * Subclasses can override this to also validate the options for their sub-programs. */ protected void validateOptions(Program program, ProgramOptions options) { // this will throw an exception if the custom tx timeout is invalid SystemArguments.validateTransactionTimeout(options.getUserArguments().asMap(), cConf); }
programRunId = programOptions.getProgramId().run(ProgramRunners.getRunId(programOptions)); Arguments systemArgs = programOptions.getArguments(); ClusterMode clusterMode = ProgramRunners.getClusterMode(programOptions); new File(systemArgs.getOption(ProgramOptionConstants.APP_SPEC_FILE)), ApplicationSpecification.class); program = Programs.create(cConf, programRunner, new ProgramDescriptor(programOptions.getProgramId(), appSpec), programJarLocation, new File(systemArgs.getOption(ProgramOptionConstants.EXPANDED_PROGRAM_JAR)));
private void addOnPremiseServices(Injector injector, ProgramOptions programOptions, MetricsCollectionService metricsCollectionService, Collection<Service> services) { services.add(injector.getInstance(KafkaClientService.class)); services.add(injector.getInstance(BrokerService.class)); services.add(new ProgramRunnableResourceReporter(programOptions.getProgramId(), metricsCollectionService, context)); }
private ProgramOptions createComponentOptions(int instanceId, int instances, RunId runId, ProgramOptions options) { Map<String, String> systemOptions = Maps.newHashMap(); systemOptions.putAll(options.getArguments().asMap()); systemOptions.put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(instanceId)); systemOptions.put(ProgramOptionConstants.INSTANCES, Integer.toString(instances)); systemOptions.put(ProgramOptionConstants.RUN_ID, runId.getId()); systemOptions.put(ProgramOptionConstants.HOST, host); return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemOptions), options.getUserArguments()); }
@Override public void start(ProgramRunId programRunId, ProgramOptions programOptions, @Nullable String twillRunId, ProgramDescriptor programDescriptor) { ImmutableMap.Builder<String, String> properties = ImmutableMap.<String, String>builder() .put(ProgramOptionConstants.PROGRAM_RUN_ID, GSON.toJson(programRunId)) .put(ProgramOptionConstants.PROGRAM_STATUS, ProgramRunStatus.STARTING.name()) .put(ProgramOptionConstants.USER_OVERRIDES, GSON.toJson(programOptions.getUserArguments().asMap())) .put(ProgramOptionConstants.SYSTEM_OVERRIDES, GSON.toJson(programOptions.getArguments().asMap())) .put(ProgramOptionConstants.PROGRAM_DESCRIPTOR, GSON.toJson(programDescriptor)); if (twillRunId != null) { properties.put(ProgramOptionConstants.TWILL_RUN_ID, twillRunId); } programStatePublisher.publish(Notification.Type.PROGRAM_STATUS, properties.build()); }
/** * Returns {@code true} if running in local mode. */ static boolean isLocal(ProgramOptions programOptions) { return !Boolean.parseBoolean(programOptions.getArguments().getOption(DISTRIBUTED_MODE)); }
public void provisioned(ProgramRunId programRunId, ProgramOptions programOptions, ProgramDescriptor programDescriptor, String userId, Cluster cluster, URI secureKeysDir) { Map<String, String> properties = ImmutableMap.<String, String>builder() .put(ProgramOptionConstants.PROGRAM_RUN_ID, GSON.toJson(programRunId)) .put(ProgramOptionConstants.PROGRAM_DESCRIPTOR, GSON.toJson(programDescriptor)) .put(ProgramOptionConstants.USER_ID, userId) .put(ProgramOptionConstants.CLUSTER_STATUS, ProgramRunClusterStatus.PROVISIONED.name()) .put(ProgramOptionConstants.CLUSTER, GSON.toJson(cluster)) .put(ProgramOptionConstants.DEBUG_ENABLED, String.valueOf(programOptions.isDebug())) .put(ProgramOptionConstants.USER_OVERRIDES, GSON.toJson(programOptions.getUserArguments().asMap())) .put(ProgramOptionConstants.SYSTEM_OVERRIDES, GSON.toJson(programOptions.getArguments().asMap())) .put(ProgramOptionConstants.SECURE_KEYS_DIR, GSON.toJson(secureKeysDir)) .build(); publish(properties); }
private static Map<String, String> createMetricsTags(ProgramOptions programOptions, @Nullable String taskId, @Nullable MapReduceMetrics.TaskType type, @Nullable WorkflowProgramInfo workflowProgramInfo) { Map<String, String> tags = Maps.newHashMap(); if (type != null) { tags.put(Constants.Metrics.Tag.MR_TASK_TYPE, type.getId()); if (taskId != null) { String taskMetricsPreference = programOptions.getUserArguments().asMap().get(SystemArguments.METRICS_CONTEXT_TASK_INCLUDED); boolean taskLevelPreference = taskMetricsPreference == null ? false : Boolean.valueOf(taskMetricsPreference); if (taskLevelPreference) { tags.put(Constants.Metrics.Tag.INSTANCE_ID, taskId); } } } if (workflowProgramInfo != null) { workflowProgramInfo.updateMetricsTags(tags); } return tags; }
programRunId = programOptions.getProgramId().run(ProgramRunners.getRunId(programOptions)); Arguments systemArgs = programOptions.getArguments(); ClusterMode clusterMode = ProgramRunners.getClusterMode(programOptions); new File(systemArgs.getOption(ProgramOptionConstants.APP_SPEC_FILE)), ApplicationSpecification.class); program = Programs.create(cConf, programRunner, new ProgramDescriptor(programOptions.getProgramId(), appSpec), programJarLocation, new File(systemArgs.getOption(ProgramOptionConstants.EXPANDED_PROGRAM_JAR)));
private void addOnPremiseServices(Injector injector, ProgramOptions programOptions, MetricsCollectionService metricsCollectionService, Collection<Service> services) { services.add(injector.getInstance(KafkaClientService.class)); services.add(injector.getInstance(BrokerService.class)); services.add(injector.getInstance(StreamCoordinatorClient.class)); services.add(new ProgramRunnableResourceReporter(programOptions.getProgramId(), metricsCollectionService, context)); }
@Override public JsonElement serialize(ProgramOptions src, Type typeOfSrc, JsonSerializationContext context) { JsonObject json = new JsonObject(); json.add("programId", context.serialize(src.getProgramId(), ProgramId.class)); json.add("arguments", context.serialize(src.getArguments(), Arguments.class)); json.add("userArguments", context.serialize(src.getUserArguments(), Arguments.class)); json.addProperty("debug", src.isDebug()); return json; } }
private ProgramOptions createComponentOptions(int instanceId, int instances, RunId runId, ProgramOptions options) { Map<String, String> systemOptions = Maps.newHashMap(); systemOptions.putAll(options.getArguments().asMap()); systemOptions.put(ProgramOptionConstants.INSTANCE_ID, Integer.toString(instanceId)); systemOptions.put(ProgramOptionConstants.INSTANCES, Integer.toString(instances)); systemOptions.put(ProgramOptionConstants.RUN_ID, runId.getId()); systemOptions.put(ProgramOptionConstants.HOST, host); return new SimpleProgramOptions(options.getProgramId(), new BasicArguments(systemOptions), options.getUserArguments()); }
@Override protected void setupLaunchConfig(ProgramLaunchConfig launchConfig, Program program, ProgramOptions options, CConfiguration cConf, Configuration hConf, File tempDir) { ApplicationSpecification appSpec = program.getApplicationSpecification(); WorkerSpecification workerSpec = appSpec.getWorkers().get(program.getName()); String instances = options.getArguments().getOption(ProgramOptionConstants.INSTANCES, String.valueOf(workerSpec.getInstances())); launchConfig.addRunnable(workerSpec.getName(), new WorkerTwillRunnable(workerSpec.getName()), Integer.parseInt(instances), options.getUserArguments().asMap(), workerSpec.getResources()); } }
@Nullable private TriggeringScheduleInfo getTriggeringScheduleInfo(ProgramOptions programOptions) { String scheduleInfoString = programOptions.getArguments().getOption(ProgramOptionConstants.TRIGGERING_SCHEDULE_INFO); return scheduleInfoString == null ? null : GSON.fromJson(scheduleInfoString, TriggeringScheduleInfo.class); }
public void provisioning(ProgramRunId programRunId, ProgramOptions programOptions, ProgramDescriptor programDescriptor, String userId) { publish(ImmutableMap.<String, String>builder() .put(ProgramOptionConstants.PROGRAM_RUN_ID, GSON.toJson(programRunId)) .put(ProgramOptionConstants.PROGRAM_DESCRIPTOR, GSON.toJson(programDescriptor)) .put(ProgramOptionConstants.USER_ID, userId) .put(ProgramOptionConstants.CLUSTER_STATUS, ProgramRunClusterStatus.PROVISIONING.name()) .put(ProgramOptionConstants.DEBUG_ENABLED, String.valueOf(programOptions.isDebug())) .put(ProgramOptionConstants.USER_OVERRIDES, GSON.toJson(programOptions.getUserArguments().asMap())) .put(ProgramOptionConstants.SYSTEM_OVERRIDES, GSON.toJson(programOptions.getArguments().asMap())) .put(ProgramOptionConstants.ARTIFACT_ID, GSON.toJson(programDescriptor.getArtifactId().toApiArtifactId())) .build()); }