@VisibleForTesting protected static Spec disableFlowRunImmediatelyOnStart(FlowSpec spec) { Properties properties = spec.getConfigAsProperties(); properties.setProperty(ConfigurationKeys.FLOW_RUN_IMMEDIATELY, "false"); Config config = ConfigFactory.parseProperties(properties); FlowSpec flowSpec = new FlowSpec(spec.getUri(), spec.getVersion(), spec.getDescription(), config, properties, spec.getTemplateURIs(), spec.getChildSpecs()); return flowSpec; }
@Test public void testDisableFlowRunImmediatelyOnStart() throws Exception { Properties properties = new Properties(); properties.setProperty(ConfigurationKeys.FLOW_RUN_IMMEDIATELY, "true"); properties.setProperty(ConfigurationKeys.JOB_SCHEDULE_KEY, TEST_SCHEDULE); properties.setProperty(ConfigurationKeys.JOB_GROUP_KEY, TEST_GROUP_NAME); properties.setProperty(ConfigurationKeys.JOB_NAME_KEY, TEST_FLOW_NAME); Config config = ConfigFactory.parseProperties(properties); FlowSpec spec = FlowSpec.builder().withTemplate(new URI(TEST_TEMPLATE_URI)).withVersion("version") .withConfigAsProperties(properties).withConfig(config).build(); FlowSpec modifiedSpec = (FlowSpec) GobblinServiceJobScheduler.disableFlowRunImmediatelyOnStart(spec); for (URI templateURI : modifiedSpec.getTemplateURIs().get()) { Assert.assertEquals(templateURI.toString(), TEST_TEMPLATE_URI); } Assert.assertEquals(modifiedSpec.getVersion(), "version"); Config modifiedConfig = modifiedSpec.getConfig(); Assert.assertFalse(modifiedConfig.getBoolean(ConfigurationKeys.FLOW_RUN_IMMEDIATELY)); Assert.assertEquals(modifiedConfig.getString(ConfigurationKeys.JOB_GROUP_KEY), TEST_GROUP_NAME); Assert.assertEquals(modifiedConfig.getString(ConfigurationKeys.JOB_NAME_KEY), TEST_FLOW_NAME); } }
public String toLongString() { return getUri().toString() + "/" + getVersion() + "[" + getDescription() + "]"; }
public String toShortString() { return getUri().toString() + "/" + getVersion(); }
JobSpec jobSpec; JobSpec.Builder jobSpecBuilder = JobSpec.builder(jobSpecURIGenerator(flowSpec, sourceNode, targetNode)) .withConfig(flowSpec.getConfig()) .withDescription(flowSpec.getDescription()) .withVersion(flowSpec.getVersion()); if (templateURI != null) { jobSpecBuilder.withTemplate(templateURI); if (flowSpec.getConfig().hasPath(ConfigurationKeys.FLOW_NAME_KEY)) { jobSpec.setConfig(jobSpec.getConfig() .withValue(ConfigurationKeys.JOB_NAME_KEY, ConfigValueFactory.fromAnyRef( flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_NAME_KEY).unwrapped().toString() + "-" + sourceNode.getNodeName() + "-" + targetNode.getNodeName()))); if (flowSpec.getConfig().hasPath(ConfigurationKeys.FLOW_GROUP_KEY)) { jobSpec.setConfig(jobSpec.getConfig() .withValue(ConfigurationKeys.JOB_GROUP_KEY, flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_GROUP_KEY)));
JobSpec jobSpec; JobSpec.Builder jobSpecBuilder = JobSpec.builder(jobSpecURIGenerator(flowSpec)) .withConfig(flowSpec.getConfig()) .withDescription(flowSpec.getDescription()) .withVersion(flowSpec.getVersion()); if (flowSpec.getTemplateURIs().isPresent() && templateCatalog.isPresent()) { jobSpecBuilder = jobSpecBuilder.withTemplate(flowSpec.getTemplateURIs().get().iterator().next()); try { jobSpec = new ResolvedJobSpec(jobSpecBuilder.build(), templateCatalog.get()); if (flowSpec.getConfig().hasPath(ConfigurationKeys.FLOW_NAME_KEY)) { jobSpec.setConfig(jobSpec.getConfig() .withValue(ConfigurationKeys.JOB_NAME_KEY, flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_NAME_KEY))); if (flowSpec.getConfig().hasPath(ConfigurationKeys.FLOW_GROUP_KEY)) { jobSpec.setConfig(jobSpec.getConfig() .withValue(ConfigurationKeys.JOB_GROUP_KEY, flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_GROUP_KEY)));
@Test(dependsOnMethods = "testAddConfig") public void testUpdateConfig() throws IOException, GitAPIException, URISyntaxException { // push an updated config file Files.write("flow.name=testFlow\nflow.group=testGroup\nparam1=value2\n", testFlowFile, Charsets.UTF_8); // add, commit, push this.gitForPush.add().addFilepattern(formConfigFilePath(this.testGroupDir.getName(), this.testFlowFile.getName())) .call(); this.gitForPush.commit().setMessage("Third commit").call(); this.gitForPush.push().setRemote("origin").setRefSpecs(this.masterRefSpec).call(); this.gitConfigMonitor.processGitConfigChanges(); Collection<Spec> specs = this.flowCatalog.getSpecs(); Assert.assertTrue(specs.size() == 1); FlowSpec spec = (FlowSpec) (specs.iterator().next()); Assert.assertEquals(spec.getUri(), new URI("gobblin-flow:/testGroup/testFlow")); Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow"); Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup"); Assert.assertEquals(spec.getConfig().getString("param1"), "value2"); }
FlowSpec flowSpec = (FlowSpec) addedSpec; Properties jobConfig = new Properties(); Properties flowSpecProperties = ((FlowSpec) addedSpec).getConfigAsProperties(); jobConfig.putAll(this.properties); jobConfig.setProperty(ConfigurationKeys.JOB_NAME_KEY, addedSpec.getUri().toString()); jobConfig.setProperty(ConfigurationKeys.JOB_GROUP_KEY, flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_GROUP_KEY).toString()); jobConfig.setProperty(ConfigurationKeys.FLOW_RUN_IMMEDIATELY, ConfigUtils.getString((flowSpec).getConfig(), ConfigurationKeys.FLOW_RUN_IMMEDIATELY, "false")); if (flowSpecProperties.containsKey(ConfigurationKeys.JOB_SCHEDULE_KEY) && StringUtils .isNotBlank(flowSpecProperties.getProperty(ConfigurationKeys.JOB_SCHEDULE_KEY))) { if (PropertiesUtils.getPropAsBoolean(jobConfig, ConfigurationKeys.FLOW_RUN_IMMEDIATELY, "false")) { _log.info("RunImmediately requested, hence executing FlowSpec: " + addedSpec); this.jobExecutor.execute(new NonScheduledJobRunner(flowSpec.getUri(), false, jobConfig, null)); this.jobExecutor.execute(new NonScheduledJobRunner(flowSpec.getUri(), true, jobConfig, null));
/** * It can receive multiple number of parameters, needed to generate a unique URI. * Implementation is flowSpecCompiler dependent. * This method should return URI which has job name at third place, when split by "/" * e.g. /flowGroup/flowName * /flowGroup/flowName/sourceNode-targetNode * SafeDatasetCommit creates state store using this name and * {@link org.apache.gobblin.runtime.job_monitor.KafkaJobMonitor} extract job name to find the state store path. * @param objects * @return */ public URI jobSpecURIGenerator(Object... objects) { return ((FlowSpec)objects[0]).getUri(); }
/** * remove a {@link FlowSpec} for a deleted or renamed flow config * @param change */ @Override public void removeChange(DiffEntry change) { if (checkConfigFilePath(change.getOldPath())) { Path configFilePath = new Path(this.repositoryDir, change.getOldPath()); String flowName = FSSpecStore.getSpecName(configFilePath); String flowGroup = FSSpecStore.getSpecGroup(configFilePath); // build a dummy config to get the proper URI for delete Config dummyConfig = ConfigBuilder.create() .addPrimitive(ConfigurationKeys.FLOW_GROUP_KEY, flowGroup) .addPrimitive(ConfigurationKeys.FLOW_NAME_KEY, flowName) .build(); FlowSpec spec = FlowSpec.builder() .withConfig(dummyConfig) .withVersion(SPEC_VERSION) .withDescription(SPEC_DESCRIPTION) .build(); this.flowCatalog.remove(spec.getUri()); } }
static Map<String, String> getFlowMetadata(FlowSpec flowSpec) { Map<String, String> metadata = Maps.newHashMap(); metadata.put(TimingEvent.FlowEventConstants.FLOW_NAME_FIELD, flowSpec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY)); metadata.put(TimingEvent.FlowEventConstants.FLOW_GROUP_FIELD, flowSpec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY)); Config flowConfig = flowSpec.getConfig(); if (flowConfig.hasPath(ConfigurationKeys.FLOW_EXECUTION_ID_KEY)) { metadata.put(TimingEvent.FlowEventConstants.FLOW_EXECUTION_ID_FIELD, flowConfig.getString(ConfigurationKeys.FLOW_EXECUTION_ID_KEY)); } return metadata; }
private FlowSpec createFlowSpec(String flowConfigResource, String source, String destination, boolean applyRetention, boolean applyRetentionOnInput) throws IOException, URISyntaxException { //Create a flow spec Properties flowProperties = new Properties(); flowProperties.put(ConfigurationKeys.JOB_SCHEDULE_KEY, "* * * * *"); flowProperties.put(ConfigurationKeys.FLOW_GROUP_KEY, "testFlowGroup"); flowProperties.put(ConfigurationKeys.FLOW_NAME_KEY, "testFlowName"); flowProperties.put(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY, source); flowProperties.put(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY, destination); flowProperties.put(ConfigurationKeys.FLOW_APPLY_RETENTION, Boolean.toString(applyRetention)); flowProperties.put(ConfigurationKeys.FLOW_APPLY_INPUT_RETENTION, Boolean.toString(applyRetentionOnInput)); Config flowConfig = ConfigUtils.propertiesToConfig(flowProperties); //Get the input/output dataset config from a file URI flowConfigUri = MultiHopFlowCompilerTest.class.getClassLoader().getResource(flowConfigResource).toURI(); Path flowConfigPath = new Path(flowConfigUri); FileSystem fs1 = FileSystem.get(flowConfigUri, new Configuration()); try (InputStream is = fs1.open(flowConfigPath)) { Config datasetConfig = ConfigFactory.parseReader(new InputStreamReader(is, Charset.defaultCharset())); flowConfig = flowConfig.withFallback(datasetConfig).resolve(); } FlowSpec.Builder flowSpecBuilder = null; flowSpecBuilder = FlowSpec.builder(new Path("/tmp/flowSpecCatalog").toUri()) .withConfig(flowConfig) .withDescription("dummy description") .withVersion(FlowSpec.Builder.DEFAULT_VERSION); FlowSpec spec = flowSpecBuilder.build(); return spec; }
FlowSpec spec = (FlowSpec) flowCatalog.getSpec(flowUri); FlowConfig flowConfig = new FlowConfig(); Properties flowProps = spec.getConfigAsProperties(); Schedule schedule = null; } else if (spec.getTemplateURIs().isPresent()) { flowConfig.setTemplateUris(StringUtils.join(spec.getTemplateURIs().get(), ",")); } else { flowConfig.setTemplateUris("NA");
Assert.assertEquals(flowSpec.getConfig().getString(FLOW_GROUP_KEY), TEST_GROUP_NAME); Assert.assertEquals(flowSpec.getConfig().getString(FLOW_NAME_KEY), TEST_FLOW_NAME); Assert.assertEquals(flowSpec.getConfig().getString(SCHEDULE_KEY), TEST_SCHEDULE); Assert.assertEquals(flowSpec.getConfig().getBoolean(RUN_IMMEDIATELY_KEY), true); Assert.assertEquals(flowSpec.getConfig().getString("param1"), "a:b:c*.d"); Assert.assertEquals(flowSpec.getTemplateURIs().get().size(), 1); Assert.assertTrue(flowSpec.getTemplateURIs().get().contains(new URI(TEST_TEMPLATE_URI))); flowSpec = FlowConfigResourceLocalHandler.createFlowSpecForConfig(flowConfig); Assert.assertEquals(flowSpec.getConfig().getString(FLOW_GROUP_KEY), TEST_GROUP_NAME); Assert.assertEquals(flowSpec.getConfig().getString(FLOW_NAME_KEY), TEST_FLOW_NAME); Assert.assertEquals(flowSpec.getConfig().getString(SCHEDULE_KEY), TEST_SCHEDULE); Assert.assertEquals(flowSpec.getConfig().getBoolean(RUN_IMMEDIATELY_KEY), true); Assert.assertEquals(flowSpec.getConfig().getString("param1"),"value1"); Assert.assertEquals(flowSpec.getConfig().getString("param2"),"value1-123"); Assert.assertEquals(flowSpec.getConfig().getString("param3"), "a:b:c*.d"); Assert.assertEquals(flowSpec.getTemplateURIs().get().size(), 1); Assert.assertTrue(flowSpec.getTemplateURIs().get().contains(new URI(TEST_TEMPLATE_URI)));
public void put(Spec spec, boolean triggerListener) { try { Preconditions.checkState(state() == State.RUNNING, String.format("%s is not running.", this.getClass().getName())); Preconditions.checkNotNull(spec); long startTime = System.currentTimeMillis(); log.info(String.format("Adding FlowSpec with URI: %s and Config: %s", spec.getUri(), ((FlowSpec) spec).getConfigAsProperties())); specStore.addSpec(spec); metrics.updatePutSpecTime(startTime); if (triggerListener) { this.listeners.onAddSpec(spec); } } catch (IOException e) { throw new RuntimeException("Cannot add Spec to Spec store: " + spec, e); } }
/** * It returns the template uri for job. * This method can be overridden by derived FlowToJobSpecCompiler classes. * @param flowSpec * @return template URI */ protected URI jobSpecTemplateURIGenerator(FlowSpec flowSpec) { // For now only first template uri will be honored for Identity return flowSpec.getTemplateURIs().get().iterator().next(); }
public FlowSpec build() { Preconditions.checkNotNull(this.uri); Preconditions.checkArgument(null != version, "Version should not be null"); return new FlowSpec(getURI(), getVersion(), getDescription(), getConfig(), getConfigAsProperties(), getTemplateURIs(), getChildSpecs()); }
Config flowConfig = flowSpec.getConfig(); .withDescription(flowSpec.getDescription()).withVersion(flowSpec.getVersion());
JobSpec jobSpec; JobSpec.Builder jobSpecBuilder = JobSpec.builder(jobSpecURIGenerator(flowSpec)) .withConfig(flowSpec.getConfig()) .withDescription(flowSpec.getDescription()) .withVersion(flowSpec.getVersion()); if (flowSpec.getTemplateURIs().isPresent() && templateCatalog.isPresent()) { jobSpecBuilder = jobSpecBuilder.withTemplate(flowSpec.getTemplateURIs().get().iterator().next()); try { jobSpec = new ResolvedJobSpec(jobSpecBuilder.build(), templateCatalog.get()); if (flowSpec.getConfig().hasPath(ConfigurationKeys.FLOW_NAME_KEY)) { jobSpec.setConfig(jobSpec.getConfig() .withValue(ConfigurationKeys.JOB_NAME_KEY, flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_NAME_KEY))); if (flowSpec.getConfig().hasPath(ConfigurationKeys.FLOW_GROUP_KEY)) { jobSpec.setConfig(jobSpec.getConfig() .withValue(ConfigurationKeys.JOB_GROUP_KEY, flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_GROUP_KEY)));
@Test public void testAddConfig() throws IOException, GitAPIException, URISyntaxException { // push a new config file this.testGroupDir.mkdirs(); this.testFlowFile.createNewFile(); Files.write("flow.name=testFlow\nflow.group=testGroup\nparam1=value1\n", testFlowFile, Charsets.UTF_8); // add, commit, push this.gitForPush.add().addFilepattern(formConfigFilePath(this.testGroupDir.getName(), this.testFlowFile.getName())) .call(); this.gitForPush.commit().setMessage("Second commit").call(); this.gitForPush.push().setRemote("origin").setRefSpecs(this.masterRefSpec).call(); this.gitConfigMonitor.processGitConfigChanges(); Collection<Spec> specs = this.flowCatalog.getSpecs(); Assert.assertTrue(specs.size() == 1); FlowSpec spec = (FlowSpec) (specs.iterator().next()); Assert.assertEquals(spec.getUri(), new URI("gobblin-flow:/testGroup/testFlow")); Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow"); Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup"); Assert.assertEquals(spec.getConfig().getString("param1"), "value1"); }