private Config getResolvedConfigHelper(Config userConfig, Set<JobTemplate> alreadyLoadedTemplates) throws SpecNotFoundException, TemplateException { Config config = getLocallyResolvedConfig(userConfig); for (JobTemplate template : Lists.reverse(this.superTemplates)) { if (!alreadyLoadedTemplates.contains(template)) { alreadyLoadedTemplates.add(template); Config fallback = template instanceof InheritingJobTemplate ? ((InheritingJobTemplate) template).getResolvedConfigHelper(config, alreadyLoadedTemplates) : template.getResolvedConfig(config); config = config.withFallback(fallback); } } return config; }
@Override public List<Config> getResolvedJobConfigs(Config userConfig, DatasetDescriptor inputDescriptor, DatasetDescriptor outputDescriptor) throws SpecNotFoundException, JobTemplate.TemplateException { Config inputDescriptorConfig = inputDescriptor.getRawConfig().atPath(DatasetDescriptorConfigKeys.FLOW_EDGE_INPUT_DATASET_DESCRIPTOR_PREFIX); Config outputDescriptorConfig = outputDescriptor.getRawConfig().atPath(DatasetDescriptorConfigKeys.FLOW_EDGE_OUTPUT_DATASET_DESCRIPTOR_PREFIX); userConfig = userConfig.withFallback(inputDescriptorConfig).withFallback(outputDescriptorConfig); List<Config> resolvedJobConfigs = new ArrayList<>(); for (JobTemplate jobTemplate: getJobTemplates()) { Config resolvedJobConfig = jobTemplate.getResolvedConfig(userConfig).resolve().withValue( ConfigurationKeys.JOB_TEMPLATE_PATH, ConfigValueFactory.fromAnyRef(jobTemplate.getUri().toString()));; resolvedJobConfigs.add(resolvedJobConfig); } return resolvedJobConfigs; } }
private Config getRawTemplateConfigHelper(Set<JobTemplate> alreadyInheritedTemplates) throws SpecNotFoundException, TemplateException { Config rawTemplate = getLocalRawTemplate(); for (JobTemplate template : Lists.reverse(this.superTemplates)) { if (!alreadyInheritedTemplates.contains(template)) { alreadyInheritedTemplates.add(template); Config thisFallback = template instanceof InheritingJobTemplate ? ((InheritingJobTemplate) template).getRawTemplateConfigHelper(alreadyInheritedTemplates) : template.getRawTemplateConfig(); rawTemplate = rawTemplate.withFallback(thisFallback); } } return rawTemplate; }
Assert.assertEquals(jobTemplates.size(), 4); for (int i = 0; i < 4; i++) { String uri = new Path(jobTemplates.get(i).getUri()).getName().split("\\.")[0]; String templateId = uri.substring(uri.length() - 1); for (int j = 0; j < 2; j++) { Config jobTemplateConfig = jobTemplates.get(i).getRawTemplateConfig(); String suffix = templateId + Integer.toString(j + 1); Assert.assertEquals(jobTemplateConfig.getString("key" + suffix), "val" + suffix);
/** * As the base implementation here, all templates will be considered for each edge. */ @Override protected void populateEdgeTemplateMap() { if (templateCatalog.isPresent()) { for (FlowEdge flowEdge : this.weightedGraph.edgeSet()) { edgeTemplateMap.put(flowEdge.getEdgeIdentity(), templateCatalog.get(). getAllTemplates(). stream().map(jobTemplate -> jobTemplate.getUri()).collect(Collectors.toList())); } } }
private Set<String> getRequiredConfigListHelper(Set<JobTemplate> alreadyLoadedTemplates) throws SpecNotFoundException, TemplateException { Set<String> requiredConfigs = Sets.newHashSet(getLocallyRequiredConfigList()); for (JobTemplate template : this.superTemplates) { if (!alreadyLoadedTemplates.contains(template)) { alreadyLoadedTemplates.add(template); requiredConfigs.addAll(template instanceof InheritingJobTemplate ? ((InheritingJobTemplate) template).getRequiredConfigListHelper(alreadyLoadedTemplates) : template.getRequiredConfigList()); } } return requiredConfigs; }
String jobSpecUri = Files.getNameWithoutExtension(new Path(jobTemplate.getUri()).getName()); jobExecutionPlans.add(new JobExecutionPlan(JobSpec.builder(jobSpecUri).withConfig(jobTemplate.getRawTemplateConfig()). withVersion("1").withTemplate(jobTemplate.getUri()).build(), specExecutor));
@Test public void test() throws Exception { Map<String, String> confMap = Maps.newHashMap(); confMap.put("key1", "value1"); confMap.put(ConfigurationKeys.REQUIRED_ATRRIBUTES_LIST, "required1,required2"); confMap.put(StaticJobTemplate.SUPER_TEMPLATE_KEY, "template2"); JobCatalogWithTemplates catalog = Mockito.mock(JobCatalogWithTemplates.class); Mockito.when(catalog.getTemplate(new URI("template2"))).thenAnswer( new InheritingJobTemplateTest.TestTemplateAnswer( Lists.<URI>newArrayList(), ImmutableMap.of("key2", "value2"), Lists.<String>newArrayList(), catalog)); StaticJobTemplate template = new StaticJobTemplate(new URI("template"), "1", "desc", ConfigFactory.parseMap(confMap), catalog); Assert.assertEquals(template.getSuperTemplates().size(), 1); Assert.assertEquals(template.getSuperTemplates().iterator().next().getUri(), new URI("template2")); Collection<String> required = template.getRequiredConfigList(); Assert.assertEquals(required.size(), 2); Assert.assertTrue(required.contains("required1")); Assert.assertTrue(required.contains("required2")); Config rawTemplate = template.getRawTemplateConfig(); Assert.assertEquals(rawTemplate.getString("key1"), "value1"); Assert.assertEquals(rawTemplate.getString("key2"), "value2"); Config resolved = template.getResolvedConfig(ConfigFactory.parseMap(ImmutableMap.of("required1", "r1", "required2", "r2"))); Assert.assertEquals(resolved.getString("key1"), "value1"); Assert.assertEquals(resolved.getString("key2"), "value2"); Assert.assertEquals(resolved.getString("required1"), "r1"); Assert.assertEquals(resolved.getString("required2"), "r2"); }
@Test public void test() throws Exception { JobCatalogWithTemplates underlying = Mockito.mock(JobCatalogWithTemplates.class); JobCatalogWithTemplates catalog = new PackagedTemplatesJobCatalogDecorator(underlying); JobTemplate classTemplate = catalog.getTemplate(new URI(PackagedTemplatesJobCatalogDecorator.CLASS + "://" + TestTemplate.class.getName())); Assert.assertEquals(classTemplate.getClass(), TestTemplate.class); try { catalog.getTemplate(new URI(PackagedTemplatesJobCatalogDecorator.CLASS + "://" + "non.existing.class")); Assert.fail(); } catch (SpecNotFoundException exc) { // expect exception } JobTemplate resourceTemplate = catalog.getTemplate(new URI(PackagedTemplatesJobCatalogDecorator.RESOURCE + ":///templates/test.template")); Assert.assertEquals(resourceTemplate.getClass(), ResourceBasedJobTemplate.class); Assert.assertEquals(resourceTemplate.getRequiredConfigList().size(), 3); URI uri = new URI("scheme:///templates/test.template"); try { catalog.getTemplate(uri); Assert.fail(); } catch (SpecNotFoundException exc) { // expect exception } Mockito.verify(underlying).getTemplate(uri); }
private static Config resolveConfig(JobSpec jobSpec, JobCatalog catalog) throws SpecNotFoundException, JobTemplate.TemplateException { Optional<URI> templateURIOpt = jobSpec.getTemplateURI(); if (templateURIOpt.isPresent()) { JobCatalogWithTemplates catalogWithTemplates = new PackagedTemplatesJobCatalogDecorator(catalog); JobTemplate template = catalogWithTemplates.getTemplate(templateURIOpt.get()); return template.getResolvedConfig(jobSpec.getConfig()).resolve(); } else { return jobSpec.getConfig().resolve(); } }
@Override public List<Config> getResolvedJobConfigs(Config userConfig, DatasetDescriptor inputDescriptor, DatasetDescriptor outputDescriptor) throws SpecNotFoundException, JobTemplate.TemplateException { Config inputDescriptorConfig = inputDescriptor.getRawConfig().atPath(DatasetDescriptorConfigKeys.FLOW_EDGE_INPUT_DATASET_DESCRIPTOR_PREFIX); Config outputDescriptorConfig = outputDescriptor.getRawConfig().atPath(DatasetDescriptorConfigKeys.FLOW_EDGE_OUTPUT_DATASET_DESCRIPTOR_PREFIX); userConfig = userConfig.withFallback(inputDescriptorConfig).withFallback(outputDescriptorConfig); List<Config> resolvedJobConfigs = new ArrayList<>(); for (JobTemplate jobTemplate: getJobTemplates()) { Config resolvedJobConfig = jobTemplate.getResolvedConfig(userConfig).resolve().withValue( ConfigurationKeys.JOB_TEMPLATE_PATH, ConfigValueFactory.fromAnyRef(jobTemplate.getUri().toString()));; resolvedJobConfigs.add(resolvedJobConfig); } return resolvedJobConfigs; } }
/** * As the base implementation here, all templates will be considered for each edge. */ @Override protected void populateEdgeTemplateMap() { if (templateCatalog.isPresent()) { for (FlowEdge flowEdge : this.weightedGraph.edgeSet()) { edgeTemplateMap.put(flowEdge.getEdgeIdentity(), templateCatalog.get(). getAllTemplates(). stream().map(jobTemplate -> jobTemplate.getUri()).collect(Collectors.toList())); } } }
private Set<String> getRequiredConfigListHelper(Set<JobTemplate> alreadyLoadedTemplates) throws SpecNotFoundException, TemplateException { Set<String> requiredConfigs = Sets.newHashSet(getLocallyRequiredConfigList()); for (JobTemplate template : this.superTemplates) { if (!alreadyLoadedTemplates.contains(template)) { alreadyLoadedTemplates.add(template); requiredConfigs.addAll(template instanceof InheritingJobTemplate ? ((InheritingJobTemplate) template).getRequiredConfigListHelper(alreadyLoadedTemplates) : template.getRequiredConfigList()); } } return requiredConfigs; }
private Config getRawTemplateConfigHelper(Set<JobTemplate> alreadyInheritedTemplates) throws SpecNotFoundException, TemplateException { Config rawTemplate = getLocalRawTemplate(); for (JobTemplate template : Lists.reverse(this.superTemplates)) { if (!alreadyInheritedTemplates.contains(template)) { alreadyInheritedTemplates.add(template); Config thisFallback = template instanceof InheritingJobTemplate ? ((InheritingJobTemplate) template).getRawTemplateConfigHelper(alreadyInheritedTemplates) : template.getRawTemplateConfig(); rawTemplate = rawTemplate.withFallback(thisFallback); } } return rawTemplate; }
URI templateUri = new URI(jobProps.getProperty(TEMPLATE_KEY)); Config resolvedJob = new PackagedTemplatesJobCatalogDecorator().getTemplate(templateUri) .getResolvedConfig(ConfigUtils.propertiesToConfig(jobProps)); jobProps = ConfigUtils.configToProperties(resolvedJob);
/** * Checks if the {@link FlowTemplate} is resolvable using the provided {@link Config} object. A {@link FlowTemplate} * is resolvable only if each of the {@link JobTemplate}s in the flow is resolvable * @param userConfig User supplied Config * @return true if the {@link FlowTemplate} is resolvable */ @Override public boolean isResolvable(Config userConfig, DatasetDescriptor inputDescriptor, DatasetDescriptor outputDescriptor) throws SpecNotFoundException, JobTemplate.TemplateException { Config inputDescriptorConfig = inputDescriptor.getRawConfig().atPath(DatasetDescriptorConfigKeys.FLOW_EDGE_INPUT_DATASET_DESCRIPTOR_PREFIX); Config outputDescriptorConfig = outputDescriptor.getRawConfig().atPath(DatasetDescriptorConfigKeys.FLOW_EDGE_OUTPUT_DATASET_DESCRIPTOR_PREFIX); userConfig = userConfig.withFallback(inputDescriptorConfig).withFallback(outputDescriptorConfig); ConfigResolveOptions resolveOptions = ConfigResolveOptions.defaults().setAllowUnresolved(true); for (JobTemplate template: this.jobTemplates) { Config templateConfig = template.getResolvedConfig(userConfig).resolve(resolveOptions); if (!template.getResolvedConfig(userConfig).resolve(resolveOptions).isResolved()) { return false; } } return true; }
if (this.template != null) { try { finalConfig = this.template.getResolvedConfig(finalConfig); } catch (SpecNotFoundException | JobTemplate.TemplateException exc) { throw new RuntimeException(exc);
private Config getResolvedConfigHelper(Config userConfig, Set<JobTemplate> alreadyLoadedTemplates) throws SpecNotFoundException, TemplateException { Config config = getLocallyResolvedConfig(userConfig); for (JobTemplate template : Lists.reverse(this.superTemplates)) { if (!alreadyLoadedTemplates.contains(template)) { alreadyLoadedTemplates.add(template); Config fallback = template instanceof InheritingJobTemplate ? ((InheritingJobTemplate) template).getResolvedConfigHelper(config, alreadyLoadedTemplates) : template.getResolvedConfig(config); config = config.withFallback(fallback); } } return config; }
private static Config resolveConfig(JobSpec jobSpec, JobCatalog catalog) throws SpecNotFoundException, JobTemplate.TemplateException { Optional<URI> templateURIOpt = jobSpec.getTemplateURI(); if (templateURIOpt.isPresent()) { JobCatalogWithTemplates catalogWithTemplates = new PackagedTemplatesJobCatalogDecorator(catalog); JobTemplate template = catalogWithTemplates.getTemplate(templateURIOpt.get()); return template.getResolvedConfig(jobSpec.getConfig()).resolve(); } else { return jobSpec.getConfig().resolve(); } }
URI templateUri = new URI(jobProps.getProperty(TEMPLATE_KEY)); Config resolvedJob = new PackagedTemplatesJobCatalogDecorator().getTemplate(templateUri) .getResolvedConfig(ConfigUtils.propertiesToConfig(jobProps)); jobProps = ConfigUtils.configToProperties(resolvedJob);