JobSpec.Builder builder = JobSpec.builder(jobConfigURI).withConfig(filteredConfig) .withDescription(description) .withVersion(version); builder.withTemplate(new URI(rawConfig.getString(ConfigurationKeys.JOB_TEMPLATE_PATH))); } catch (URISyntaxException e) { throw new RuntimeException("Bad job template URI " + e, e); return builder.build();
/** Creates a builder for the JobSpec based on values in a job properties config. */ public static Builder builder(URI catalogURI, Properties jobProps) { String name = JobState.getJobNameFromProps(jobProps); String group = JobState.getJobGroupFromProps(jobProps); if (null == group) { group = "default"; } try { URI jobURI = new URI(catalogURI.getScheme(), catalogURI.getAuthority(), "/" + group + "/" + name, null); Builder builder = new Builder(jobURI).withConfigAsProperties(jobProps); String descr = JobState.getJobDescriptionFromProps(jobProps); if (null != descr) { builder.withDescription(descr); } return builder; } catch (URISyntaxException e) { throw new RuntimeException("Unable to create a JobSpec URI: " + e, e); } }
public URI getDefaultURI() { URI jobCatalogURI = getJobCatalogURI(); Config jobCfg = getConfig(); String name = jobCfg.hasPath(ConfigurationKeys.JOB_NAME_KEY) ? jobCfg.getString(ConfigurationKeys.JOB_NAME_KEY) : "default"; String group = jobCfg.hasPath(ConfigurationKeys.JOB_GROUP_KEY) ? jobCfg.getString(ConfigurationKeys.JOB_GROUP_KEY) : "default"; try { return new URI(jobCatalogURI.getScheme(), jobCatalogURI.getAuthority(), "/" + group + "/" + name, null, null); } catch (URISyntaxException e) { throw new RuntimeException("Unable to create default JobSpec URI:" + e, e); } }
jobSpec = this.specBuilder.withConfig(finalConfig).build();
jobSpecBuilder.withJobCatalogURI(record.getUri()).withVersion(record.getVersion()) .withDescription(record.getDescription()).withConfigAsProperties(props); jobSpecBuilder.withTemplate(new URI(record.getTemplateUri())); Verb verb = Verb.valueOf(verbName); changesSpecs.add(new ImmutablePair<Verb, Spec>(verb, jobSpecBuilder.build())); } catch (Throwable t) { _log.error("Could not decode record at partition " + this.currentPartitionIdx +
.withConfig(flowSpec.getConfig()) .withDescription(flowSpec.getDescription()) .withVersion(flowSpec.getVersion()); jobSpecBuilder = jobSpecBuilder.withTemplate(flowSpec.getTemplateURIs().get().iterator().next()); try { jobSpec = new ResolvedJobSpec(jobSpecBuilder.build(), templateCatalog.get()); log.info("Resolved JobSpec properties are: " + jobSpec.getConfigAsProperties()); } catch (SpecNotFoundException | JobTemplate.TemplateException e) { jobSpec = jobSpecBuilder.build(); log.info("Unresolved JobSpec properties are: " + jobSpec.getConfigAsProperties());
jobSpecBuilder.withJobCatalogURI(record.getUri()).withVersion(record.getVersion()) .withDescription(record.getDescription()).withConfigAsProperties(props); jobSpecBuilder.withTemplate(new URI(record.getTemplateUri())); } catch (URISyntaxException e) { log.error("could not parse template URI " + record.getTemplateUri()); Verb verb = Verb.valueOf(verbName); JobSpec jobSpec = jobSpecBuilder.build();
@Override public Collection<Either<JobSpec, URI>> parseJobSpec(GobblinTrackingEvent event) { if (!acceptEvent(event)) { this.rejectedEvents.inc(); return Lists.newArrayList(); } String datasetURN = event.getMetadata().get(SlaEventKeys.DATASET_URN_KEY); URI jobSpecURI = PathUtils.mergePaths(new Path(this.baseURI), new Path(datasetURN)).toUri(); Map<String, String> jobConfigMap = Maps.newHashMap(); for (Map.Entry<String, String> entry : this.extractKeys.entrySet()) { if (event.getMetadata().containsKey(entry.getKey())) { jobConfigMap.put(entry.getValue(), event.getMetadata().get(entry.getKey())); } } Config jobConfig = ConfigFactory.parseMap(jobConfigMap); JobSpec jobSpec = JobSpec.builder(jobSpecURI).withTemplate(this.template).withConfig(jobConfig).build(); return Lists.newArrayList(Either.<JobSpec, URI>left(jobSpec)); }
@CliObjectSupport(argumentNames = {"jobName"}) public EmbeddedGobblin(String name) { HadoopUtils.addGobblinSite(); this.specBuilder = new JobSpec.Builder(name); this.userConfigMap = Maps.newHashMap(); this.builtConfigMap = Maps.newHashMap(); this.sysConfigOverrides = Maps.newHashMap(); this.defaultSysConfig = getDefaultSysConfig(); this.distributedJars = Maps.newHashMap(); loadCoreGobblinJarsToDistributedJars(); this.distributeJarsFunction = new Runnable() { @Override public void run() { // NOOP } }; }
public static Builder builder(String jobSpecUri) { return new Builder(jobSpecUri); }
public static Builder builder() { return new Builder(); }
public static Builder builder(URI jobSpecUri) { return new Builder(jobSpecUri); }
@Override public void onDeleteJob(URI deletedJobURI, String deletedJobVersion) { super.onDeleteJob(deletedJobURI, deletedJobVersion); try { JobSpec.Builder jobSpecBuilder = JobSpec.builder(deletedJobURI); Properties props = new Properties(); jobSpecBuilder.withVersion(deletedJobVersion).withConfigAsProperties(props); _jobSpecQueue.put(new ImmutablePair<Verb, Spec>(Verb.DELETE, jobSpecBuilder.build())); } catch (InterruptedException e) { Thread.currentThread().interrupt(); } }
public JobSpec build() { Preconditions.checkNotNull(this.uri); Preconditions.checkNotNull(this.version); return new JobSpec(getURI(), getVersion(), getDescription(), getConfig(), getConfigAsProperties(), getTemplateURI()); }
/** * {@inheritDoc} * * NOTE: For this callback only conditions on the URI and version will be used. * */ @Override public void onDeleteJob(URI deletedJobURI, String deletedJobVersion) { JobSpec fakeJobSpec = JobSpec.builder(deletedJobURI).withVersion(deletedJobVersion).build(); if (this.filter.apply(fakeJobSpec)) { this.delegate.onDeleteJob(deletedJobURI, deletedJobVersion); } }