/** * The job template name is derived from the {@link org.apache.gobblin.runtime.api.JobTemplate} URI. It is the * simple name of the path component of the URI. * @param jobExecutionPlan * @return the simple name of the job template from the URI of its path. */ private static String getJobTemplateName(JobExecutionPlan jobExecutionPlan) { Optional<URI> jobTemplateUri = jobExecutionPlan.getJobSpec().getTemplateURI(); if (jobTemplateUri.isPresent()) { return Files.getNameWithoutExtension(new Path(jobTemplateUri.get()).getName()); } else { return null; } }
.put(ImmutableFSJobCatalog.VERSION_KEY_IN_JOBSPEC, jobSpec.getVersion()); if (jobSpec.getTemplateURI().isPresent()) { mapBuilder.put(ConfigurationKeys.JOB_TEMPLATE_PATH, jobSpec.getTemplateURI().get().toString());
private AvroJobSpec convertToAvroJobSpec(Spec spec, SpecExecutor.Verb verb) { if (spec instanceof JobSpec) { JobSpec jobSpec = (JobSpec) spec; AvroJobSpec.Builder avroJobSpecBuilder = AvroJobSpec.newBuilder(); avroJobSpecBuilder.setUri(jobSpec.getUri().toString()).setVersion(jobSpec.getVersion()) .setDescription(jobSpec.getDescription()).setProperties(Maps.fromProperties(jobSpec.getConfigAsProperties())) .setMetadata(ImmutableMap.of(VERB_KEY, verb.name())); if (jobSpec.getTemplateURI().isPresent()) { avroJobSpecBuilder.setTemplateUri(jobSpec.getTemplateURI().get().toString()); } return avroJobSpecBuilder.build(); } else { throw new RuntimeException("Unsupported spec type " + spec.getClass()); } } }
private static Config resolveConfig(JobSpec jobSpec, JobCatalog catalog) throws SpecNotFoundException, JobTemplate.TemplateException { Optional<URI> templateURIOpt = jobSpec.getTemplateURI(); if (templateURIOpt.isPresent()) { JobCatalogWithTemplates catalogWithTemplates = new PackagedTemplatesJobCatalogDecorator(catalog); JobTemplate template = catalogWithTemplates.getTemplate(templateURIOpt.get()); return template.getResolvedConfig(jobSpec.getConfig()).resolve(); } else { return jobSpec.getConfig().resolve(); } }
jobSpecJson.addProperty(SerializationConstants.JOB_SPEC_VERSION_KEY, jobSpec.getVersion()); jobSpecJson.addProperty(SerializationConstants.JOB_SPEC_DESCRIPTION_KEY, jobSpec.getDescription()); String jobSpecTemplateURI = (jobSpec.getTemplateURI().isPresent()) ? jobSpec.getTemplateURI().get().toString() : null; jobSpecJson.addProperty(SerializationConstants.JOB_SPEC_TEMPLATE_URI_KEY, jobSpecTemplateURI); jobSpecJson.addProperty(SerializationConstants.JOB_SPEC_CONFIG_KEY, jobSpec.getConfig().root().render(ConfigRenderOptions.concise()));
@Test public void testParseJobSpec() throws Exception { SLAEventKafkaJobMonitor monitor = new SLAEventKafkaJobMonitor("topic", null, new URI("/base/URI"), HighLevelConsumerTest.getSimpleConfig(Optional.of(KafkaJobMonitor.KAFKA_JOB_MONITOR_PREFIX)), new NoopSchemaVersionWriter(), Optional.<Pattern>absent(), Optional.<Pattern>absent(), this.templateURI, ImmutableMap.of("metadataKey1", "key1")); monitor.buildMetricsContextAndMetrics(); GobblinTrackingEvent event = createSLAEvent("DatasetPublish", new URI("/data/myDataset"), ImmutableMap.of("metadataKey1","value1","key1","value2")); Collection<Either<JobSpec, URI>> jobSpecs = monitor.parseJobSpec(event); Assert.assertEquals(jobSpecs.size(), 1); JobSpec jobSpec = (JobSpec) jobSpecs.iterator().next().get(); Assert.assertEquals(jobSpec.getUri(), new URI("/base/URI/data/myDataset")); Assert.assertEquals(jobSpec.getTemplateURI().get(), templateURI); // should insert configuration from metadata Assert.assertEquals(jobSpec.getConfig().getString("key1"), "value1"); monitor.shutdownMetrics(); }
/** * Resolve the job spec using classpath templates as well as any templates available in the input {@link JobCatalog}. */ public ResolvedJobSpec(JobSpec other, JobCatalog catalog) throws SpecNotFoundException, JobTemplate.TemplateException { super(other.getUri(), other.getVersion(), other.getDescription(), resolveConfig(other, catalog), ConfigUtils.configToProperties(resolveConfig(other, catalog)), other.getTemplateURI(), other.getMetadata()); this.originalJobSpec = other; }
String startNodeName = new Path(dag.getStartNodes().get(0).getValue().getJobSpec().getUri()).getName(); Assert.assertEquals(startNodeName, "job1"); String templateUri = new Path(dag.getStartNodes().get(0).getValue().getJobSpec().getTemplateURI().get()).getName(); Assert.assertEquals(templateUri, "job1.job"); String endNodeName = new Path(dag.getEndNodes().get(0).getValue().getJobSpec().getUri()).getName(); Assert.assertEquals(endNodeName, "job4"); templateUri = new Path(dag.getEndNodes().get(0).getValue().getJobSpec().getTemplateURI().get()).getName(); Assert.assertEquals(templateUri, "job4.job");
/** * The job template name is derived from the {@link org.apache.gobblin.runtime.api.JobTemplate} URI. It is the * simple name of the path component of the URI. * @param jobExecutionPlan * @return the simple name of the job template from the URI of its path. */ private static String getJobTemplateName(JobExecutionPlan jobExecutionPlan) { Optional<URI> jobTemplateUri = jobExecutionPlan.getJobSpec().getTemplateURI(); if (jobTemplateUri.isPresent()) { return Files.getNameWithoutExtension(new Path(jobTemplateUri.get()).getName()); } else { return null; } }
.put(ImmutableFSJobCatalog.VERSION_KEY_IN_JOBSPEC, jobSpec.getVersion()); if (jobSpec.getTemplateURI().isPresent()) { mapBuilder.put(ConfigurationKeys.JOB_TEMPLATE_PATH, jobSpec.getTemplateURI().get().toString());
private AvroJobSpec convertToAvroJobSpec(Spec spec, SpecExecutor.Verb verb) { if (spec instanceof JobSpec) { JobSpec jobSpec = (JobSpec) spec; AvroJobSpec.Builder avroJobSpecBuilder = AvroJobSpec.newBuilder(); avroJobSpecBuilder.setUri(jobSpec.getUri().toString()).setVersion(jobSpec.getVersion()) .setDescription(jobSpec.getDescription()).setProperties(Maps.fromProperties(jobSpec.getConfigAsProperties())) .setMetadata(ImmutableMap.of(VERB_KEY, verb.name())); if (jobSpec.getTemplateURI().isPresent()) { avroJobSpecBuilder.setTemplateUri(jobSpec.getTemplateURI().get().toString()); } return avroJobSpecBuilder.build(); } else { throw new RuntimeException("Unsupported spec type " + spec.getClass()); } } }
private static Config resolveConfig(JobSpec jobSpec, JobCatalog catalog) throws SpecNotFoundException, JobTemplate.TemplateException { Optional<URI> templateURIOpt = jobSpec.getTemplateURI(); if (templateURIOpt.isPresent()) { JobCatalogWithTemplates catalogWithTemplates = new PackagedTemplatesJobCatalogDecorator(catalog); JobTemplate template = catalogWithTemplates.getTemplate(templateURIOpt.get()); return template.getResolvedConfig(jobSpec.getConfig()).resolve(); } else { return jobSpec.getConfig().resolve(); } }
jobSpecJson.addProperty(SerializationConstants.JOB_SPEC_VERSION_KEY, jobSpec.getVersion()); jobSpecJson.addProperty(SerializationConstants.JOB_SPEC_DESCRIPTION_KEY, jobSpec.getDescription()); String jobSpecTemplateURI = (jobSpec.getTemplateURI().isPresent()) ? jobSpec.getTemplateURI().get().toString() : null; jobSpecJson.addProperty(SerializationConstants.JOB_SPEC_TEMPLATE_URI_KEY, jobSpecTemplateURI); jobSpecJson.addProperty(SerializationConstants.JOB_SPEC_CONFIG_KEY, jobSpec.getConfig().root().render(ConfigRenderOptions.concise()));
/** * Resolve the job spec using classpath templates as well as any templates available in the input {@link JobCatalog}. */ public ResolvedJobSpec(JobSpec other, JobCatalog catalog) throws SpecNotFoundException, JobTemplate.TemplateException { super(other.getUri(), other.getVersion(), other.getDescription(), resolveConfig(other, catalog), ConfigUtils.configToProperties(resolveConfig(other, catalog)), other.getTemplateURI(), other.getMetadata()); this.originalJobSpec = other; }