public static ResourceBasedJobTemplate forResourcePath(String path) throws SpecNotFoundException, TemplateException, IOException, URISyntaxException { return forResourcePath(path, new PackagedTemplatesJobCatalogDecorator()); }
public static ResourceBasedJobTemplate forResourcePath(String path, JobCatalogWithTemplates catalog) throws SpecNotFoundException, TemplateException, IOException, URISyntaxException { return forURI(new URI(path), catalog); }
@Test public void test() throws Exception { ResourceBasedJobTemplate template = ResourceBasedJobTemplate.forResourcePath("templates/test.template", null); Collection<String> required = template.getRequiredConfigList(); Assert.assertEquals(required.size(), 3); Assert.assertTrue(required.contains("required0")); Assert.assertTrue(required.contains("required1")); Assert.assertTrue(required.contains("required2")); Config rawTemplate = template.getRawTemplateConfig(); Assert.assertEquals(rawTemplate.getString("templated0"), "x"); Assert.assertEquals(rawTemplate.getString("templated1"), "y"); Config resolved = template.getResolvedConfig( ConfigFactory.parseMap(ImmutableMap.of("required0", "r0", "required1", "r1", "required2", "r2"))); Assert.assertEquals(resolved.getString("templated0"), "x"); Assert.assertEquals(resolved.getString("required0"), "r0"); }
public static ResourceBasedJobTemplate forURI(URI uri, JobCatalogWithTemplates catalog) throws SpecNotFoundException, TemplateException, IOException { try (InputStream is = getInputStreamForURI(uri)) { return new ResourceBasedJobTemplate(is, uri, catalog); } }
private static Properties resolveTemplate(Properties jobProps) throws IOException { try { if (jobProps.containsKey(ConfigurationKeys.JOB_TEMPLATE_PATH)) { Config jobConfig = ConfigUtils.propertiesToConfig(jobProps); Properties resolvedProps = ConfigUtils.configToProperties((ResourceBasedJobTemplate .forResourcePath(jobProps.getProperty(ConfigurationKeys.JOB_TEMPLATE_PATH), new PackagedTemplatesJobCatalogDecorator())) .getResolvedConfig(jobConfig)); return resolvedProps; } else { return jobProps; } } catch (JobTemplate.TemplateException | SpecNotFoundException | URISyntaxException exc) { throw new IOException(exc); } } }
@Test public void testRequiredAttrList() throws Exception { Properties jobProps = this.userProp; Collection<String> requiredConfigList = ResourceBasedJobTemplate.forURI(new URI( jobProps.getProperty(ConfigurationKeys.JOB_TEMPLATE_PATH)), new PackagedTemplatesJobCatalogDecorator()) .getRequiredConfigList(); Assert.assertEquals(requiredConfigList.size(), 3); Assert.assertTrue( requiredConfigList.contains("required0")); Assert.assertTrue( requiredConfigList.contains("required1")); Assert.assertTrue( requiredConfigList.contains("required2")); }
template = (ResourceBasedJobTemplate) catalog.getTemplate(templateURI.toUri()); templateConfig = sysConfig.withFallback(template.getRawTemplateConfig()).withFallback(baseConfig).resolve(configResolveOptions); } catch (SpecNotFoundException|JobTemplate.TemplateException exc) { throw new IOException(exc);
@Test public void testResolvingConfig() throws Exception { Config jobProps = ConfigFactory.parseProperties(this.userProp); Assert.assertEquals(ConfigUtils.configToProperties(jobProps).size(), 6); jobProps = ResourceBasedJobTemplate.forResourcePath(jobProps.getString(ConfigurationKeys.JOB_TEMPLATE_PATH), new PackagedTemplatesJobCatalogDecorator()) .getResolvedConfig(jobProps); // Remove job.template in userSpecified file and gobblin.template.required_attributes in template Assert.assertEquals(ConfigUtils.configToProperties(jobProps).size(), 8); Properties targetResolvedJobProps = new Properties() ; targetResolvedJobProps.setProperty("a", "1"); targetResolvedJobProps.setProperty("templated0", "2"); targetResolvedJobProps.setProperty("templated1", "y"); targetResolvedJobProps.setProperty("required0","r0"); targetResolvedJobProps.setProperty("required1","r1"); targetResolvedJobProps.setProperty("required2","r2"); targetResolvedJobProps.setProperty(ConfigurationKeys.JOB_TEMPLATE_PATH, "templates/test.template"); targetResolvedJobProps.setProperty(ConfigurationKeys.REQUIRED_ATRRIBUTES_LIST, "required0,required1,required2"); Assert.assertEquals(targetResolvedJobProps, ConfigUtils.configToProperties(jobProps)); }
public static ResourceBasedJobTemplate forURI(URI uri, JobCatalogWithTemplates catalog) throws SpecNotFoundException, TemplateException, IOException { try (InputStream is = getInputStreamForURI(uri)) { return new ResourceBasedJobTemplate(is, uri, catalog); } }
template = (ResourceBasedJobTemplate) catalog.getTemplate(templateURI.toUri()); templateConfig = sysConfig.withFallback(template.getRawTemplateConfig()).withFallback(baseConfig).resolve(configResolveOptions); } catch (SpecNotFoundException|JobTemplate.TemplateException exc) { throw new IOException(exc);
public EmbeddedGobblinDistcp(Path from, Path to) throws JobTemplate.TemplateException, IOException { super("Distcp"); try { setTemplate(ResourceBasedJobTemplate.forResourcePath("templates/distcp.template")); } catch (URISyntaxException | SpecNotFoundException exc) { throw new RuntimeException("Could not instantiate an " + EmbeddedGobblinDistcp.class.getName(), exc); } this.setConfiguration("from", from.toString()); this.setConfiguration("to", to.toString()); // Infer source and target fs uris from the input paths this.setConfiguration(ConfigurationKeys.SOURCE_FILEBASED_FS_URI, from.getFileSystem(new Configuration()).getUri().toString()); this.setConfiguration(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, to.getFileSystem(new Configuration()).getUri().toString()); // add gobblin-data-management jar to distributed jars this.distributeJar(ClassUtil.findContainingJar(CopySource.class)); }
private static Properties resolveTemplate(Properties jobProps) throws IOException { try { if (jobProps.containsKey(ConfigurationKeys.JOB_TEMPLATE_PATH)) { Config jobConfig = ConfigUtils.propertiesToConfig(jobProps); Properties resolvedProps = ConfigUtils.configToProperties((ResourceBasedJobTemplate .forResourcePath(jobProps.getProperty(ConfigurationKeys.JOB_TEMPLATE_PATH), new PackagedTemplatesJobCatalogDecorator())) .getResolvedConfig(jobConfig)); return resolvedProps; } else { return jobProps; } } catch (JobTemplate.TemplateException | SpecNotFoundException | URISyntaxException exc) { throw new IOException(exc); } } }
@Override public JobTemplate getTemplate(URI uri) throws SpecNotFoundException, JobTemplate.TemplateException { if (RESOURCE.equals(uri.getScheme())) { try { // resources are accessed by relative uris, make the uri relative (get path, strip initial /) URI actualResourceUri = new URI(uri.getPath().substring(1)); return ResourceBasedJobTemplate.forURI(actualResourceUri, this); } catch (URISyntaxException use) { throw new RuntimeException("Error when computing resource path.", use); } catch (IOException ioe) { throw new SpecNotFoundException(uri, ioe); } } else if(CLASS.equals(uri.getScheme())) { try { return ((Class<? extends JobTemplate>) Class.forName(uri.getAuthority())).newInstance(); } catch (ReflectiveOperationException roe) { throw new SpecNotFoundException(uri, roe); } } if (this.underlying != null && this.underlying instanceof JobCatalogWithTemplates) { JobTemplate template = ((JobCatalogWithTemplates) this.underlying).getTemplate(uri); if (template == null) { throw new SpecNotFoundException(uri); } return template; } throw new SpecNotFoundException(uri); }
@CliObjectSupport(argumentNames = {"topics"}) public EmbeddedWikipediaExample(String... topics) throws JobTemplate.TemplateException, IOException { super("Wikipedia"); try { setTemplate(ResourceBasedJobTemplate.forResourcePath("wikipedia.template")); } catch (URISyntaxException | SpecNotFoundException exc) { throw new RuntimeException("Could not instantiate an " + EmbeddedWikipediaExample.class.getName(), exc); } this.setConfiguration("titles", Joiner.on(",").join(topics)); }
public static ResourceBasedJobTemplate forResourcePath(String path, JobCatalogWithTemplates catalog) throws SpecNotFoundException, TemplateException, IOException, URISyntaxException { return forURI(new URI(path), catalog); }
setTemplate(ResourceBasedJobTemplate.forResourcePath("githubjsontoparquet.template")); } catch (URISyntaxException | SpecNotFoundException e) { e.printStackTrace();
@Override public JobTemplate getTemplate(URI uri) throws SpecNotFoundException, JobTemplate.TemplateException { if (RESOURCE.equals(uri.getScheme())) { try { // resources are accessed by relative uris, make the uri relative (get path, strip initial /) URI actualResourceUri = new URI(uri.getPath().substring(1)); return ResourceBasedJobTemplate.forURI(actualResourceUri, this); } catch (URISyntaxException use) { throw new RuntimeException("Error when computing resource path.", use); } catch (IOException ioe) { throw new SpecNotFoundException(uri, ioe); } } else if(CLASS.equals(uri.getScheme())) { try { return ((Class<? extends JobTemplate>) Class.forName(uri.getAuthority())).newInstance(); } catch (ReflectiveOperationException roe) { throw new SpecNotFoundException(uri, roe); } } if (this.underlying != null && this.underlying instanceof JobCatalogWithTemplates) { JobTemplate template = ((JobCatalogWithTemplates) this.underlying).getTemplate(uri); if (template == null) { throw new SpecNotFoundException(uri); } return template; } throw new SpecNotFoundException(uri); }
@Test public void testRunWithTemplate() throws Exception { String eventBusId = this.getClass().getName(); int numHellos = 5; TestingEventBusAsserter asserter = new TestingEventBusAsserter(eventBusId); EmbeddedGobblin embeddedGobblin = new EmbeddedGobblin("TestJob").setTemplate(ResourceBasedJobTemplate.forResourcePath("templates/hello-world.template")); embeddedGobblin.setConfiguration(ConfigurationKeys.WRITER_BUILDER_CLASS, GobblinTestEventBusWriter.Builder.class.getName()); embeddedGobblin.setConfiguration(GobblinTestEventBusWriter.FULL_EVENTBUSID_KEY, eventBusId); embeddedGobblin.setConfiguration(HelloWorldSource.NUM_HELLOS_FULL_KEY, Integer.toString(numHellos)); JobExecutionResult result = embeddedGobblin.run(); Assert.assertTrue(result.isSuccessful()); ArrayList<String> expectedEvents = new ArrayList<>(); for (int i = 1; i <= numHellos; ++i) { expectedEvents.add(HelloWorldSource.ExtractorImpl.helloMessage(i)); } asserter.assertNextValuesEq(expectedEvents); asserter.close(); }
public static ResourceBasedJobTemplate forResourcePath(String path) throws SpecNotFoundException, TemplateException, IOException, URISyntaxException { return forResourcePath(path, new PackagedTemplatesJobCatalogDecorator()); }
public EmbeddedGobblinDistcp(Path from, Path to) throws JobTemplate.TemplateException, IOException { super("Distcp"); try { setTemplate(ResourceBasedJobTemplate.forResourcePath("templates/distcp.template")); } catch (URISyntaxException | SpecNotFoundException exc) { throw new RuntimeException("Could not instantiate an " + EmbeddedGobblinDistcp.class.getName(), exc); } this.setConfiguration("from", from.toString()); this.setConfiguration("to", to.toString()); // Infer source and target fs uris from the input paths this.setConfiguration(ConfigurationKeys.SOURCE_FILEBASED_FS_URI, from.getFileSystem(new Configuration()).getUri().toString()); this.setConfiguration(ConfigurationKeys.WRITER_FILE_SYSTEM_URI, to.getFileSystem(new Configuration()).getUri().toString()); // add gobblin-data-management jar to distributed jars this.distributeJar(ClassUtil.findContainingJar(CopySource.class)); }