builder = (templateURI == null) ? builder : builder.withTemplate(new URI(templateURI)); builder = (version == null) ? builder : builder.withVersion(version); builder = (description == null) ? builder : builder.withDescription(description); jobSpec = builder.withConfig(jobConfig).build(); } catch (URISyntaxException e) {
.withDescription(description) .withVersion(version);
/** Creates a builder for the JobSpec based on values in a job properties config. */ public static Builder builder(URI catalogURI, Properties jobProps) { String name = JobState.getJobNameFromProps(jobProps); String group = JobState.getJobGroupFromProps(jobProps); if (null == group) { group = "default"; } try { URI jobURI = new URI(catalogURI.getScheme(), catalogURI.getAuthority(), "/" + group + "/" + name, null); Builder builder = new Builder(jobURI).withConfigAsProperties(jobProps); String descr = JobState.getJobDescriptionFromProps(jobProps); if (null != descr) { builder.withDescription(descr); } return builder; } catch (URISyntaxException e) { throw new RuntimeException("Unable to create a JobSpec URI: " + e, e); } }
.withDescription(flowSpec.getDescription()).withVersion(flowSpec.getVersion());
JobSpec.Builder jobSpecBuilder = JobSpec.builder(jobSpecURIGenerator(flowSpec, sourceNode, targetNode)) .withConfig(flowSpec.getConfig()) .withDescription(flowSpec.getDescription()) .withVersion(flowSpec.getVersion()); if (templateURI != null) {
.withDescription(description) .withVersion(version);
JobSpec.Builder jobSpecBuilder = JobSpec.builder(jobSpecURIGenerator(flowSpec)) .withConfig(flowSpec.getConfig()) .withDescription(flowSpec.getDescription()) .withVersion(flowSpec.getVersion());
@Override public Dag<JobExecutionPlan> compileFlow(Spec spec) { List<JobExecutionPlan> jobExecutionPlans = new ArrayList<>(); long flowExecutionId = System.currentTimeMillis(); int i = 0; while(i++ < NUMBER_OF_JOBS) { String specUri = "/foo/bar/spec/" + i; Properties properties = new Properties(); properties.put(ConfigurationKeys.FLOW_NAME_KEY, ((FlowSpec)spec).getConfigAsProperties().get(ConfigurationKeys.FLOW_NAME_KEY)); properties.put(ConfigurationKeys.FLOW_GROUP_KEY, ((FlowSpec)spec).getConfigAsProperties().get(ConfigurationKeys.FLOW_GROUP_KEY)); properties.put(ConfigurationKeys.JOB_NAME_KEY, ((FlowSpec)spec).getConfigAsProperties().get(ConfigurationKeys.FLOW_NAME_KEY) + "_" + i); properties.put(ConfigurationKeys.JOB_GROUP_KEY, ((FlowSpec)spec).getConfigAsProperties().get(ConfigurationKeys.FLOW_GROUP_KEY) + "_" + i); properties.put(ConfigurationKeys.FLOW_EXECUTION_ID_KEY, flowExecutionId); JobSpec jobSpec = JobSpec.builder(specUri) .withConfig(ConfigUtils.propertiesToConfig(properties)) .withVersion("1") .withDescription("Spec Description") .build(); jobExecutionPlans.add(new JobExecutionPlan(jobSpec, new InMemorySpecExecutor(ConfigFactory.empty()))); } return new JobExecutionPlanDagFactory().createDag(jobExecutionPlans); } }
/** * Creates {@link JobSpec} from the {@link AvroJobSpec} record. * @param record the record as an {@link AvroJobSpec} * @return a {@link JobSpec} wrapped in a {@link Collection} of {@link Either} */ @Override public Collection<Either<JobSpec, URI>> parseJobSpec(AvroJobSpec record) { JobSpec.Builder jobSpecBuilder = JobSpec.builder(record.getUri()); Properties props = new Properties(); props.putAll(record.getProperties()); jobSpecBuilder.withJobCatalogURI(record.getUri()).withVersion(record.getVersion()) .withDescription(record.getDescription()).withConfigAsProperties(props).withMetadata(record.getMetadata()); if (!record.getTemplateUri().isEmpty()) { try { jobSpecBuilder.withTemplate(new URI(record.getTemplateUri())); } catch (URISyntaxException e) { log.error("could not parse template URI " + record.getTemplateUri()); } } JobSpec jobSpec = jobSpecBuilder.build(); log.info("Parsed job spec " + jobSpec.toString()); return Lists.newArrayList(Either.<JobSpec, URI>left(jobSpec)); }
/** Creates a builder for the JobSpec based on values in a job properties config. */ public static Builder builder(URI catalogURI, Properties jobProps) { String name = JobState.getJobNameFromProps(jobProps); String group = JobState.getJobGroupFromProps(jobProps); if (null == group) { group = "default"; } try { URI jobURI = new URI(catalogURI.getScheme(), catalogURI.getAuthority(), "/" + group + "/" + name, null); Builder builder = new Builder(jobURI).withConfigAsProperties(jobProps); String descr = JobState.getJobDescriptionFromProps(jobProps); if (null != descr) { builder.withDescription(descr); } return builder; } catch (URISyntaxException e) { throw new RuntimeException("Unable to create a JobSpec URI: " + e, e); } }
props.putAll(record.getProperties()); jobSpecBuilder.withJobCatalogURI(record.getUri()).withVersion(record.getVersion()) .withDescription(record.getDescription()).withConfigAsProperties(props);
.withDescription("A test job") .withConfigAsProperties(props); .withDescription("A test job") .withConfig(cfg);
.withDescription(flowSpec.getDescription()).withVersion(flowSpec.getVersion());
JobSpec.Builder jobSpecBuilder = JobSpec.builder(jobSpecURIGenerator(flowSpec, sourceNode, targetNode)) .withConfig(flowSpec.getConfig()) .withDescription(flowSpec.getDescription()) .withVersion(flowSpec.getVersion()); if (templateURI != null) {
JobSpec.Builder jobSpecBuilder = JobSpec.builder(jobSpecURIGenerator(flowSpec)) .withConfig(flowSpec.getConfig()) .withDescription(flowSpec.getDescription()) .withVersion(flowSpec.getVersion());
.withDescription("A test job") .withConfigAsProperties(props);
@Override public Dag<JobExecutionPlan> compileFlow(Spec spec) { List<JobExecutionPlan> jobExecutionPlans = new ArrayList<>(); long flowExecutionId = System.currentTimeMillis(); int i = 0; while(i++ < NUMBER_OF_JOBS) { String specUri = "/foo/bar/spec/" + i; Properties properties = new Properties(); properties.put(ConfigurationKeys.FLOW_NAME_KEY, ((FlowSpec)spec).getConfigAsProperties().get(ConfigurationKeys.FLOW_NAME_KEY)); properties.put(ConfigurationKeys.FLOW_GROUP_KEY, ((FlowSpec)spec).getConfigAsProperties().get(ConfigurationKeys.FLOW_GROUP_KEY)); properties.put(ConfigurationKeys.JOB_NAME_KEY, ((FlowSpec)spec).getConfigAsProperties().get(ConfigurationKeys.FLOW_NAME_KEY) + "_" + i); properties.put(ConfigurationKeys.JOB_GROUP_KEY, ((FlowSpec)spec).getConfigAsProperties().get(ConfigurationKeys.FLOW_GROUP_KEY) + "_" + i); properties.put(ConfigurationKeys.FLOW_EXECUTION_ID_KEY, flowExecutionId); JobSpec jobSpec = JobSpec.builder(specUri) .withConfig(ConfigUtils.propertiesToConfig(properties)) .withVersion("1") .withDescription("Spec Description") .build(); jobExecutionPlans.add(new JobExecutionPlan(jobSpec, new InMemorySpecExecutor(ConfigFactory.empty()))); } return new JobExecutionPlanDagFactory().createDag(jobExecutionPlans); } }
/** * Creates {@link JobSpec} from the {@link AvroJobSpec} record. * @param record the record as an {@link AvroJobSpec} * @return a {@link JobSpec} wrapped in a {@link Collection} of {@link Either} */ @Override public Collection<Either<JobSpec, URI>> parseJobSpec(AvroJobSpec record) { JobSpec.Builder jobSpecBuilder = JobSpec.builder(record.getUri()); Properties props = new Properties(); props.putAll(record.getProperties()); jobSpecBuilder.withJobCatalogURI(record.getUri()).withVersion(record.getVersion()) .withDescription(record.getDescription()).withConfigAsProperties(props).withMetadata(record.getMetadata()); if (!record.getTemplateUri().isEmpty()) { try { jobSpecBuilder.withTemplate(new URI(record.getTemplateUri())); } catch (URISyntaxException e) { log.error("could not parse template URI " + record.getTemplateUri()); } } JobSpec jobSpec = jobSpecBuilder.build(); log.info("Parsed job spec " + jobSpec.toString()); return Lists.newArrayList(Either.<JobSpec, URI>left(jobSpec)); }
private JobSpec initJobSpec(String specUri) { Properties properties = new Properties(); return JobSpec.builder(specUri) .withConfig(ConfigUtils.propertiesToConfig(properties)) .withVersion("1") .withDescription("Spec Description") .build(); }
private JobSpec initJobSpec(String specUri) { Properties properties = new Properties(); return JobSpec.builder(specUri) .withConfig(ConfigUtils.propertiesToConfig(properties)) .withVersion("1") .withDescription("Spec Description") .build(); }