private FlowSpec createFlowSpec(String flowConfigResource, String source, String destination, boolean applyRetention, boolean applyRetentionOnInput) throws IOException, URISyntaxException { //Create a flow spec Properties flowProperties = new Properties(); flowProperties.put(ConfigurationKeys.JOB_SCHEDULE_KEY, "* * * * *"); flowProperties.put(ConfigurationKeys.FLOW_GROUP_KEY, "testFlowGroup"); flowProperties.put(ConfigurationKeys.FLOW_NAME_KEY, "testFlowName"); flowProperties.put(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY, source); flowProperties.put(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY, destination); flowProperties.put(ConfigurationKeys.FLOW_APPLY_RETENTION, Boolean.toString(applyRetention)); flowProperties.put(ConfigurationKeys.FLOW_APPLY_INPUT_RETENTION, Boolean.toString(applyRetentionOnInput)); Config flowConfig = ConfigUtils.propertiesToConfig(flowProperties); //Get the input/output dataset config from a file URI flowConfigUri = MultiHopFlowCompilerTest.class.getClassLoader().getResource(flowConfigResource).toURI(); Path flowConfigPath = new Path(flowConfigUri); FileSystem fs1 = FileSystem.get(flowConfigUri, new Configuration()); try (InputStream is = fs1.open(flowConfigPath)) { Config datasetConfig = ConfigFactory.parseReader(new InputStreamReader(is, Charset.defaultCharset())); flowConfig = flowConfig.withFallback(datasetConfig).resolve(); } FlowSpec.Builder flowSpecBuilder = null; flowSpecBuilder = FlowSpec.builder(new Path("/tmp/flowSpecCatalog").toUri()) .withConfig(flowConfig) .withDescription("dummy description") .withVersion(FlowSpec.Builder.DEFAULT_VERSION); FlowSpec spec = flowSpecBuilder.build(); return spec; }
@Test public void testDisableFlowRunImmediatelyOnStart() throws Exception { Properties properties = new Properties(); properties.setProperty(ConfigurationKeys.FLOW_RUN_IMMEDIATELY, "true"); properties.setProperty(ConfigurationKeys.JOB_SCHEDULE_KEY, TEST_SCHEDULE); properties.setProperty(ConfigurationKeys.JOB_GROUP_KEY, TEST_GROUP_NAME); properties.setProperty(ConfigurationKeys.JOB_NAME_KEY, TEST_FLOW_NAME); Config config = ConfigFactory.parseProperties(properties); FlowSpec spec = FlowSpec.builder().withTemplate(new URI(TEST_TEMPLATE_URI)).withVersion("version") .withConfigAsProperties(properties).withConfig(config).build(); FlowSpec modifiedSpec = (FlowSpec) GobblinServiceJobScheduler.disableFlowRunImmediatelyOnStart(spec); for (URI templateURI : modifiedSpec.getTemplateURIs().get()) { Assert.assertEquals(templateURI.toString(), TEST_TEMPLATE_URI); } Assert.assertEquals(modifiedSpec.getVersion(), "version"); Config modifiedConfig = modifiedSpec.getConfig(); Assert.assertFalse(modifiedConfig.getBoolean(ConfigurationKeys.FLOW_RUN_IMMEDIATELY)); Assert.assertEquals(modifiedConfig.getString(ConfigurationKeys.JOB_GROUP_KEY), TEST_GROUP_NAME); Assert.assertEquals(modifiedConfig.getString(ConfigurationKeys.JOB_NAME_KEY), TEST_FLOW_NAME); } }
/** * Add a {@link FlowSpec} for an added, updated, or modified flow config * @param change */ @Override public void addChange(DiffEntry change) { if (checkConfigFilePath(change.getNewPath())) { Path configFilePath = new Path(this.repositoryDir, change.getNewPath()); try { Config flowConfig = loadConfigFileWithFlowNameOverrides(configFilePath); this.flowCatalog.put(FlowSpec.builder() .withConfig(flowConfig) .withVersion(SPEC_VERSION) .withDescription(SPEC_DESCRIPTION) .build()); } catch (IOException e) { log.warn("Could not load config file: " + configFilePath); } } }
private FlowSpec initFlowSpec(String flowGroup, String flowName, String source, String destination) { Properties properties = new Properties(); properties.put(ConfigurationKeys.JOB_SCHEDULE_KEY, "* * * * *"); properties.put(ConfigurationKeys.FLOW_GROUP_KEY, flowGroup); properties.put(ConfigurationKeys.FLOW_NAME_KEY, flowName); properties.put(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY, source); properties.put(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY, destination); Config config = ConfigUtils.propertiesToConfig(properties); FlowSpec.Builder flowSpecBuilder = null; try { flowSpecBuilder = FlowSpec.builder(computeTopologySpecURI(SPEC_STORE_PARENT_DIR, FLOW_SPEC_STORE_DIR)) .withConfig(config) .withDescription("dummy description") .withVersion(SPEC_VERSION) .withTemplate(new URI(TEST_TEMPLATE_URI)); } catch (URISyntaxException e) { throw new RuntimeException(e); } return flowSpecBuilder.build(); }
private FlowSpec initFlowSpec(String flowGroup, String flowName, String source, String destination) { Properties properties = new Properties(); properties.put(ConfigurationKeys.JOB_SCHEDULE_KEY, "* * * * *"); properties.put(ConfigurationKeys.FLOW_GROUP_KEY, flowGroup); properties.put(ConfigurationKeys.FLOW_NAME_KEY, flowName); properties.put(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY, source); properties.put(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY, destination); Config config = ConfigUtils.propertiesToConfig(properties); FlowSpec.Builder flowSpecBuilder = null; try { flowSpecBuilder = FlowSpec.builder(computeTopologySpecURI(SPEC_STORE_PARENT_DIR, FLOW_SPEC_STORE_DIR)) .withConfig(config) .withDescription("dummy description") .withVersion(SPEC_VERSION) .withTemplate(new URI(TEST_TEMPLATE_URI)); } catch (URISyntaxException e) { throw new RuntimeException(e); } return flowSpecBuilder.build(); }
private FlowSpec initFlowSpec() { Properties properties = new Properties(); properties.put("specStore.fs.dir", FLOW_SPEC_STORE_DIR); properties.put("specExecInstance.capabilities", "source:destination"); properties.put("gobblin.flow.sourceIdentifier", "source"); properties.put("gobblin.flow.destinationIdentifier", "destination"); Config config = ConfigUtils.propertiesToConfig(properties); FlowSpec.Builder flowSpecBuilder = null; try { flowSpecBuilder = FlowSpec.builder(computeTopologySpecURI(SPEC_STORE_PARENT_DIR, FLOW_SPEC_GROUP_DIR)) .withConfig(config) .withDescription(SPEC_DESCRIPTION) .withVersion(SPEC_VERSION) .withTemplate(new URI("templateURI")); } catch (URISyntaxException e) { throw new RuntimeException(e); } return flowSpecBuilder.build(); }
private FlowSpec initFlowSpec() { Properties properties = new Properties(); properties.put("specStore.fs.dir", SPEC_STORE_DIR); properties.put("specExecInstance.capabilities", "source:destination"); Config config = ConfigUtils.propertiesToConfig(properties); SpecExecutor specExecutorInstanceProducer = new InMemorySpecExecutor(config); FlowSpec.Builder flowSpecBuilder = null; try { flowSpecBuilder = FlowSpec.builder(computeFlowSpecURI()) .withConfig(config) .withDescription(SPEC_DESCRIPTION) .withVersion(SPEC_VERSION) .withTemplate(new URI("templateURI")); } catch (URISyntaxException e) { throw new RuntimeException(e); } return flowSpecBuilder.build(); }
/** * remove a {@link FlowSpec} for a deleted or renamed flow config * @param change */ @Override public void removeChange(DiffEntry change) { if (checkConfigFilePath(change.getOldPath())) { Path configFilePath = new Path(this.repositoryDir, change.getOldPath()); String flowName = FSSpecStore.getSpecName(configFilePath); String flowGroup = FSSpecStore.getSpecGroup(configFilePath); // build a dummy config to get the proper URI for delete Config dummyConfig = ConfigBuilder.create() .addPrimitive(ConfigurationKeys.FLOW_GROUP_KEY, flowGroup) .addPrimitive(ConfigurationKeys.FLOW_NAME_KEY, flowName) .build(); FlowSpec spec = FlowSpec.builder() .withConfig(dummyConfig) .withVersion(SPEC_VERSION) .withDescription(SPEC_DESCRIPTION) .build(); this.flowCatalog.remove(spec.getUri()); } }
return FlowSpec.builder().withConfig(configWithFallback).withTemplate(templateURI).build(); } catch (URISyntaxException e) { throw new FlowConfigLoggedException(HttpStatus.S_400_BAD_REQUEST, "bad URI " + flowConfig.getTemplateUris(), e);
/** * Add a {@link FlowSpec} for an added, updated, or modified flow config * @param change */ @Override public void addChange(DiffEntry change) { if (checkConfigFilePath(change.getNewPath())) { Path configFilePath = new Path(this.repositoryDir, change.getNewPath()); try { Config flowConfig = loadConfigFileWithFlowNameOverrides(configFilePath); this.flowCatalog.put(FlowSpec.builder() .withConfig(flowConfig) .withVersion(SPEC_VERSION) .withDescription(SPEC_DESCRIPTION) .build()); } catch (IOException e) { log.warn("Could not load config file: " + configFilePath); } } }
/** * remove a {@link FlowSpec} for a deleted or renamed flow config * @param change */ @Override public void removeChange(DiffEntry change) { if (checkConfigFilePath(change.getOldPath())) { Path configFilePath = new Path(this.repositoryDir, change.getOldPath()); String flowName = FSSpecStore.getSpecName(configFilePath); String flowGroup = FSSpecStore.getSpecGroup(configFilePath); // build a dummy config to get the proper URI for delete Config dummyConfig = ConfigBuilder.create() .addPrimitive(ConfigurationKeys.FLOW_GROUP_KEY, flowGroup) .addPrimitive(ConfigurationKeys.FLOW_NAME_KEY, flowName) .build(); FlowSpec spec = FlowSpec.builder() .withConfig(dummyConfig) .withVersion(SPEC_VERSION) .withDescription(SPEC_DESCRIPTION) .build(); this.flowCatalog.remove(spec.getUri()); } }