@Override public Collection<Either<JobSpec, URI>> parseJobSpec(GobblinTrackingEvent event) { if (!acceptEvent(event)) { this.rejectedEvents.inc(); return Lists.newArrayList(); } String datasetURN = event.getMetadata().get(SlaEventKeys.DATASET_URN_KEY); URI jobSpecURI = PathUtils.mergePaths(new Path(this.baseURI), new Path(datasetURN)).toUri(); Map<String, String> jobConfigMap = Maps.newHashMap(); for (Map.Entry<String, String> entry : this.extractKeys.entrySet()) { if (event.getMetadata().containsKey(entry.getKey())) { jobConfigMap.put(entry.getValue(), event.getMetadata().get(entry.getKey())); } } Config jobConfig = ConfigFactory.parseMap(jobConfigMap); JobSpec jobSpec = JobSpec.builder(jobSpecURI).withTemplate(this.template).withConfig(jobConfig).build(); return Lists.newArrayList(Either.<JobSpec, URI>left(jobSpec)); }
try { JobSpec.Builder builder = (uri == null) ? JobSpec.builder() : JobSpec.builder(uri); builder = (templateURI == null) ? builder : builder.withTemplate(new URI(templateURI)); builder = (version == null) ? builder : builder.withVersion(version); builder = (description == null) ? builder : builder.withDescription(description);
builder.withTemplate(new URI(rawConfig.getString(ConfigurationKeys.JOB_TEMPLATE_PATH))); } catch (URISyntaxException e) { throw new RuntimeException("Bad job template URI " + e, e);
.withVersion(flowSpec.getVersion()); if (templateURI != null) { jobSpecBuilder.withTemplate(templateURI); try { jobSpec = new ResolvedJobSpec(jobSpecBuilder.build(), templateCatalog.get());
builder.withTemplate(new URI(rawConfig.getString(ConfigurationKeys.JOB_TEMPLATE_PATH))); } catch (URISyntaxException e) { throw new RuntimeException("Bad job template URI " + e, e);
jobSpecBuilder = jobSpecBuilder.withTemplate(flowSpec.getTemplateURIs().get().iterator().next()); try { jobSpec = new ResolvedJobSpec(jobSpecBuilder.build(), templateCatalog.get());
/** * Creates {@link JobSpec} from the {@link AvroJobSpec} record. * @param record the record as an {@link AvroJobSpec} * @return a {@link JobSpec} wrapped in a {@link Collection} of {@link Either} */ @Override public Collection<Either<JobSpec, URI>> parseJobSpec(AvroJobSpec record) { JobSpec.Builder jobSpecBuilder = JobSpec.builder(record.getUri()); Properties props = new Properties(); props.putAll(record.getProperties()); jobSpecBuilder.withJobCatalogURI(record.getUri()).withVersion(record.getVersion()) .withDescription(record.getDescription()).withConfigAsProperties(props).withMetadata(record.getMetadata()); if (!record.getTemplateUri().isEmpty()) { try { jobSpecBuilder.withTemplate(new URI(record.getTemplateUri())); } catch (URISyntaxException e) { log.error("could not parse template URI " + record.getTemplateUri()); } } JobSpec jobSpec = jobSpecBuilder.build(); log.info("Parsed job spec " + jobSpec.toString()); return Lists.newArrayList(Either.<JobSpec, URI>left(jobSpec)); }
@Override public Collection<Either<JobSpec, URI>> parseJobSpec(GobblinTrackingEvent event) { if (!acceptEvent(event)) { this.rejectedEvents.inc(); return Lists.newArrayList(); } String datasetURN = event.getMetadata().get(SlaEventKeys.DATASET_URN_KEY); URI jobSpecURI = PathUtils.mergePaths(new Path(this.baseURI), new Path(datasetURN)).toUri(); Map<String, String> jobConfigMap = Maps.newHashMap(); for (Map.Entry<String, String> entry : this.extractKeys.entrySet()) { if (event.getMetadata().containsKey(entry.getKey())) { jobConfigMap.put(entry.getValue(), event.getMetadata().get(entry.getKey())); } } Config jobConfig = ConfigFactory.parseMap(jobConfigMap); JobSpec jobSpec = JobSpec.builder(jobSpecURI).withTemplate(this.template).withConfig(jobConfig).build(); return Lists.newArrayList(Either.<JobSpec, URI>left(jobSpec)); }
JobSpec js1_2 = JobSpec.builder("test_job1").withVersion("2").build(); JobSpec js2 = JobSpec.builder("test_job2").withVersion("1").build(); JobSpec js3 = JobSpec.builder("test_job3").withVersion("1").withTemplate(new URI("FS:///job3.template")) .withConfig(ConfigBuilder.create().addPrimitive("job.template", "FS:///job3.template").build()).build();
JobSpec js1_2 = JobSpec.builder("test_job1").withVersion("2").build(); JobSpec js2 = JobSpec.builder("test_job2").withVersion("1").build(); JobSpec js3 = JobSpec.builder("test_job3").withVersion("1").withTemplate(new URI("FS:///job3.template")) .withConfig(ConfigBuilder.create().addPrimitive("job.template", "FS:///job3.template").build()).build();
jobSpecBuilder.withTemplate(new URI(record.getTemplateUri()));
JobSpec jobSpec = jobSpecBuilder.withTemplate(jobTemplateUri).build();
.withVersion(flowSpec.getVersion()); if (templateURI != null) { jobSpecBuilder.withTemplate(templateURI); try { jobSpec = new ResolvedJobSpec(jobSpecBuilder.build(), templateCatalog.get());
.withTemplate(new URI("resource:///gobblin/runtime/instance/SimpleHelloWorldJob.template")) .build();
jobSpecBuilder = jobSpecBuilder.withTemplate(flowSpec.getTemplateURIs().get().iterator().next()); try { jobSpec = new ResolvedJobSpec(jobSpecBuilder.build(), templateCatalog.get());
withTemplate(new URI("job" + suffix)).build(); SpecExecutor specExecutor = InMemorySpecExecutor.createDummySpecExecutor(new URI("job" + i)); JobExecutionPlan jobExecutionPlan = new JobExecutionPlan(js, specExecutor);
String jobSpecUri = Files.getNameWithoutExtension(new Path(jobTemplate.getUri()).getName()); jobExecutionPlans.add(new JobExecutionPlan(JobSpec.builder(jobSpecUri).withConfig(jobTemplate.getRawTemplateConfig()). withVersion("1").withTemplate(jobTemplate.getUri()).build(), specExecutor));
/** * A method to create a {@link Dag <JobExecutionPlan>}. * @return a Dag. */ public Dag<JobExecutionPlan> buildDag(int numNodes, int startNodeId, boolean isForkable) throws URISyntaxException { List<JobExecutionPlan> jobExecutionPlans = new ArrayList<>(); Config baseConfig = ConfigBuilder.create(). addPrimitive(ConfigurationKeys.FLOW_GROUP_KEY, "group0"). addPrimitive(ConfigurationKeys.FLOW_NAME_KEY, "flow0"). addPrimitive(ConfigurationKeys.FLOW_EXECUTION_ID_KEY, System.currentTimeMillis()). addPrimitive(ConfigurationKeys.JOB_GROUP_KEY, "group0").build(); for (int i = startNodeId; i < startNodeId + numNodes; i++) { String suffix = Integer.toString(i); Config jobConfig = baseConfig.withValue(ConfigurationKeys.JOB_NAME_KEY, ConfigValueFactory.fromAnyRef("job" + suffix)); if (isForkable && (i == startNodeId + numNodes - 1)) { jobConfig = jobConfig.withValue(ConfigurationKeys.JOB_FORK_ON_CONCAT, ConfigValueFactory.fromAnyRef(true)); } if (i > startNodeId) { jobConfig = jobConfig.withValue(ConfigurationKeys.JOB_DEPENDENCIES, ConfigValueFactory.fromAnyRef("job" + (i - 1))); } JobSpec js = JobSpec.builder("test_job" + suffix).withVersion(suffix).withConfig(jobConfig). withTemplate(new URI("job" + suffix)).build(); SpecExecutor specExecutor = InMemorySpecExecutor.createDummySpecExecutor(new URI("job" + i)); JobExecutionPlan jobExecutionPlan = new JobExecutionPlan(js, specExecutor); jobExecutionPlans.add(jobExecutionPlan); } return new JobExecutionPlanDagFactory().createDag(jobExecutionPlans); }
/** * Create a {@link Dag<JobExecutionPlan>} with one parent and one child. * @return a Dag. */ public Dag<JobExecutionPlan> buildDag(String id, Long flowExecutionId) throws URISyntaxException { List<JobExecutionPlan> jobExecutionPlans = new ArrayList<>(); for (int i = 0; i < 2; i++) { String suffix = Integer.toString(i); Config jobConfig = ConfigBuilder.create(). addPrimitive(ConfigurationKeys.FLOW_GROUP_KEY, "group" + id). addPrimitive(ConfigurationKeys.FLOW_NAME_KEY, "flow" + id). addPrimitive(ConfigurationKeys.FLOW_EXECUTION_ID_KEY, flowExecutionId). addPrimitive(ConfigurationKeys.JOB_NAME_KEY, "job" + suffix).build(); if (i > 0) { jobConfig = jobConfig.withValue(ConfigurationKeys.JOB_DEPENDENCIES, ConfigValueFactory.fromAnyRef("job" + (i - 1))); } JobSpec js = JobSpec.builder("test_job" + suffix).withVersion(suffix).withConfig(jobConfig). withTemplate(new URI("job" + suffix)).build(); SpecExecutor specExecutor = new InMemorySpecExecutor(ConfigFactory.empty()); JobExecutionPlan jobExecutionPlan = new JobExecutionPlan(js, specExecutor); jobExecutionPlan.setExecutionStatus(ExecutionStatus.RUNNING); jobExecutionPlans.add(jobExecutionPlan); } return new JobExecutionPlanDagFactory().createDag(jobExecutionPlans); }
/** * Creates {@link JobSpec} from the {@link AvroJobSpec} record. * @param record the record as an {@link AvroJobSpec} * @return a {@link JobSpec} wrapped in a {@link Collection} of {@link Either} */ @Override public Collection<Either<JobSpec, URI>> parseJobSpec(AvroJobSpec record) { JobSpec.Builder jobSpecBuilder = JobSpec.builder(record.getUri()); Properties props = new Properties(); props.putAll(record.getProperties()); jobSpecBuilder.withJobCatalogURI(record.getUri()).withVersion(record.getVersion()) .withDescription(record.getDescription()).withConfigAsProperties(props).withMetadata(record.getMetadata()); if (!record.getTemplateUri().isEmpty()) { try { jobSpecBuilder.withTemplate(new URI(record.getTemplateUri())); } catch (URISyntaxException e) { log.error("could not parse template URI " + record.getTemplateUri()); } } JobSpec jobSpec = jobSpecBuilder.build(); log.info("Parsed job spec " + jobSpec.toString()); return Lists.newArrayList(Either.<JobSpec, URI>left(jobSpec)); }