static Map<String, String> getFlowMetadata(FlowSpec flowSpec) { Map<String, String> metadata = Maps.newHashMap(); metadata.put(TimingEvent.FlowEventConstants.FLOW_NAME_FIELD, flowSpec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY)); metadata.put(TimingEvent.FlowEventConstants.FLOW_GROUP_FIELD, flowSpec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY)); Config flowConfig = flowSpec.getConfig(); if (flowConfig.hasPath(ConfigurationKeys.FLOW_EXECUTION_ID_KEY)) { metadata.put(TimingEvent.FlowEventConstants.FLOW_EXECUTION_ID_FIELD, flowConfig.getString(ConfigurationKeys.FLOW_EXECUTION_ID_KEY)); } return metadata; }
jobConfig.setProperty(ConfigurationKeys.JOB_NAME_KEY, addedSpec.getUri().toString()); jobConfig.setProperty(ConfigurationKeys.JOB_GROUP_KEY, flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_GROUP_KEY).toString()); jobConfig.setProperty(ConfigurationKeys.FLOW_RUN_IMMEDIATELY, ConfigUtils.getString((flowSpec).getConfig(), ConfigurationKeys.FLOW_RUN_IMMEDIATELY, "false")); if (flowSpecProperties.containsKey(ConfigurationKeys.JOB_SCHEDULE_KEY) && StringUtils .isNotBlank(flowSpecProperties.getProperty(ConfigurationKeys.JOB_SCHEDULE_KEY))) {
/** * A FlowSpec contains a FlowExecutionId if it is a runOnce flow. * Refer {@link FlowConfigResourceLocalHandler#createFlowSpecForConfig} for details. * @param spec flow spec * @return flow execution id */ public static long getOrCreateFlowExecutionId(FlowSpec spec) { long flowExecutionId; if (spec.getConfig().hasPath(ConfigurationKeys.FLOW_EXECUTION_ID_KEY)) { flowExecutionId = spec.getConfig().getLong(ConfigurationKeys.FLOW_EXECUTION_ID_KEY); } else { flowExecutionId = System.currentTimeMillis(); } return flowExecutionId; } }
/**{@inheritDoc}**/ @Override public FlowGraphPath findPath(FlowSpec flowSpec) throws PathFinder.PathFinderException, ReflectiveOperationException { try { rwLock.readLock().lock(); //Instantiate a PathFinder. Class pathFinderClass = Class.forName(ConfigUtils .getString(flowSpec.getConfig(), FlowGraphConfigurationKeys.FLOW_GRAPH_PATH_FINDER_CLASS, FlowGraphConfigurationKeys.DEFAULT_FLOW_GRAPH_PATH_FINDER_CLASS)); PathFinder pathFinder = (PathFinder) GobblinConstructorUtils.invokeLongestConstructor(pathFinderClass, this, flowSpec); return pathFinder.findPath(); } finally { rwLock.readLock().unlock(); } } }
@Override public Dag<JobExecutionPlan> compileFlow(Spec spec) { Preconditions.checkNotNull(spec); Preconditions.checkArgument(spec instanceof FlowSpec, "IdentityFlowToJobSpecCompiler only converts FlowSpec to JobSpec"); long startTime = System.nanoTime(); FlowSpec flowSpec = (FlowSpec) spec; String source = flowSpec.getConfig().getString(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY); String destination = flowSpec.getConfig().getString(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY); log.info(String.format("Compiling flow for source: %s and destination: %s", source, destination)); JobSpec jobSpec = jobSpecGenerator(flowSpec); Instrumented.markMeter(this.flowCompilationSuccessFulMeter); Instrumented.updateTimer(this.flowCompilationTimer, System.nanoTime() - startTime, TimeUnit.NANOSECONDS); List<JobExecutionPlan> jobExecutionPlans; try { jobExecutionPlans = getJobExecutionPlans(source, destination, jobSpec); } catch (InterruptedException | ExecutionException e) { Instrumented.markMeter(this.flowCompilationFailedMeter); throw new RuntimeException("Cannot determine topology capabilities", e); } return new JobExecutionPlanDagFactory().createDag(jobExecutionPlans); }
private void findPath(Map<Spec, SpecExecutor> specExecutorInstanceMap, Spec spec) { inMemoryWeightGraphGenerator(); FlowSpec flowSpec = (FlowSpec) spec; if (optionalUserSpecifiedPath.isPresent()) { log.info("Starting to evaluate user's specified path ... "); if (userSpecifiedPathVerificator(specExecutorInstanceMap, flowSpec)) { log.info("User specified path[ " + optionalUserSpecifiedPath.get() + "] successfully verified."); return; } else { log.error("Will not execute user specified path[ " + optionalUserSpecifiedPath.get() + "]"); log.info("Start to execute FlowCompiler's algorithm for valid data movement path"); } } ServiceNode sourceNode = new BaseServiceNodeImpl(flowSpec.getConfig().getString(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY)); ServiceNode targetNode = new BaseServiceNodeImpl(flowSpec.getConfig().getString(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY)); List<FlowEdge> resultEdgePath = dijkstraBasedPathFindingHelper(sourceNode, targetNode, this.weightedGraph); for (int i = 0; i < resultEdgePath.size() ; i++) { FlowEdge tmpFlowEdge = resultEdgePath.get(i); ServiceNode edgeSrcNode = ((LoadBasedFlowEdgeImpl) tmpFlowEdge).getSourceNode(); ServiceNode edgeTgtNode = ((LoadBasedFlowEdgeImpl) tmpFlowEdge).getTargetNode(); specExecutorInstanceMap.put(convertHopToJobSpec(edgeSrcNode, edgeTgtNode, flowSpec), ((LoadBasedFlowEdgeImpl) (resultEdgePath.get(i))).getSpecExecutorInstance()); } }
String source = ConfigUtils.getString(flowSpec.getConfig(), ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY, ""); String destination = ConfigUtils.getString(flowSpec.getConfig(), ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY, ""); log.info(String.format("Compiling flow for source: %s and destination: %s", source, destination));
@Test(dependsOnMethods = "testAddConfig") public void testUpdateConfig() throws IOException, GitAPIException, URISyntaxException { // push an updated config file Files.write("flow.name=testFlow\nflow.group=testGroup\nparam1=value2\n", testFlowFile, Charsets.UTF_8); // add, commit, push this.gitForPush.add().addFilepattern(formConfigFilePath(this.testGroupDir.getName(), this.testFlowFile.getName())) .call(); this.gitForPush.commit().setMessage("Third commit").call(); this.gitForPush.push().setRemote("origin").setRefSpecs(this.masterRefSpec).call(); this.gitConfigMonitor.processGitConfigChanges(); Collection<Spec> specs = this.flowCatalog.getSpecs(); Assert.assertTrue(specs.size() == 1); FlowSpec spec = (FlowSpec) (specs.iterator().next()); Assert.assertEquals(spec.getUri(), new URI("gobblin-flow:/testGroup/testFlow")); Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow"); Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup"); Assert.assertEquals(spec.getConfig().getString("param1"), "value2"); }
@Test public void testDisableFlowRunImmediatelyOnStart() throws Exception { Properties properties = new Properties(); properties.setProperty(ConfigurationKeys.FLOW_RUN_IMMEDIATELY, "true"); properties.setProperty(ConfigurationKeys.JOB_SCHEDULE_KEY, TEST_SCHEDULE); properties.setProperty(ConfigurationKeys.JOB_GROUP_KEY, TEST_GROUP_NAME); properties.setProperty(ConfigurationKeys.JOB_NAME_KEY, TEST_FLOW_NAME); Config config = ConfigFactory.parseProperties(properties); FlowSpec spec = FlowSpec.builder().withTemplate(new URI(TEST_TEMPLATE_URI)).withVersion("version") .withConfigAsProperties(properties).withConfig(config).build(); FlowSpec modifiedSpec = (FlowSpec) GobblinServiceJobScheduler.disableFlowRunImmediatelyOnStart(spec); for (URI templateURI : modifiedSpec.getTemplateURIs().get()) { Assert.assertEquals(templateURI.toString(), TEST_TEMPLATE_URI); } Assert.assertEquals(modifiedSpec.getVersion(), "version"); Config modifiedConfig = modifiedSpec.getConfig(); Assert.assertFalse(modifiedConfig.getBoolean(ConfigurationKeys.FLOW_RUN_IMMEDIATELY)); Assert.assertEquals(modifiedConfig.getString(ConfigurationKeys.JOB_GROUP_KEY), TEST_GROUP_NAME); Assert.assertEquals(modifiedConfig.getString(ConfigurationKeys.JOB_NAME_KEY), TEST_FLOW_NAME); } }
@Test public void testAddConfig() throws IOException, GitAPIException, URISyntaxException { // push a new config file this.testGroupDir.mkdirs(); this.testFlowFile.createNewFile(); Files.write("flow.name=testFlow\nflow.group=testGroup\nparam1=value1\n", testFlowFile, Charsets.UTF_8); // add, commit, push this.gitForPush.add().addFilepattern(formConfigFilePath(this.testGroupDir.getName(), this.testFlowFile.getName())) .call(); this.gitForPush.commit().setMessage("Second commit").call(); this.gitForPush.push().setRemote("origin").setRefSpecs(this.masterRefSpec).call(); this.gitConfigMonitor.processGitConfigChanges(); Collection<Spec> specs = this.flowCatalog.getSpecs(); Assert.assertTrue(specs.size() == 1); FlowSpec spec = (FlowSpec) (specs.iterator().next()); Assert.assertEquals(spec.getUri(), new URI("gobblin-flow:/testGroup/testFlow")); Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow"); Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup"); Assert.assertEquals(spec.getConfig().getString("param1"), "value1"); }
@Test (dependsOnMethods = "testDelete") public void testGitCreate() throws Exception { // push a new config file File testFlowFile = new File(GIT_CLONE_DIR + "/gobblin-config/testGroup/testFlow.pull"); testFlowFile.getParentFile().mkdirs(); Files.write("flow.name=testFlow\nflow.group=testGroup\nparam1=value20\n", testFlowFile, Charsets.UTF_8); Collection<Spec> specs = this.gobblinServiceManager.flowCatalog.getSpecs(); Assert.assertTrue(specs.size() == 0); // add, commit, push this.gitForPush.add().addFilepattern("gobblin-config/testGroup/testFlow.pull").call(); this.gitForPush.commit().setMessage("second commit").call(); this.gitForPush.push().setRemote("origin").setRefSpecs(new RefSpec("master")).call(); // polling is every 5 seconds, so wait twice as long and check TimeUnit.SECONDS.sleep(10); specs = this.gobblinServiceManager.flowCatalog.getSpecs(); Assert.assertTrue(specs.size() == 1); FlowSpec spec = (FlowSpec) (specs.iterator().next()); Assert.assertEquals(spec.getUri(), new URI("gobblin-flow:/testGroup/testFlow")); Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow"); Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup"); Assert.assertEquals(spec.getConfig().getString("param1"), "value20"); }
Assert.assertEquals(flowSpec.getConfig().getString(FLOW_GROUP_KEY), TEST_GROUP_NAME); Assert.assertEquals(flowSpec.getConfig().getString(FLOW_NAME_KEY), TEST_FLOW_NAME); Assert.assertEquals(flowSpec.getConfig().getString(SCHEDULE_KEY), TEST_SCHEDULE); Assert.assertEquals(flowSpec.getConfig().getBoolean(RUN_IMMEDIATELY_KEY), true); Assert.assertEquals(flowSpec.getConfig().getString("param1"), "a:b:c*.d"); Assert.assertEquals(flowSpec.getTemplateURIs().get().size(), 1); Assert.assertTrue(flowSpec.getTemplateURIs().get().contains(new URI(TEST_TEMPLATE_URI))); flowSpec = FlowConfigResourceLocalHandler.createFlowSpecForConfig(flowConfig); Assert.assertEquals(flowSpec.getConfig().getString(FLOW_GROUP_KEY), TEST_GROUP_NAME); Assert.assertEquals(flowSpec.getConfig().getString(FLOW_NAME_KEY), TEST_FLOW_NAME); Assert.assertEquals(flowSpec.getConfig().getString(SCHEDULE_KEY), TEST_SCHEDULE); Assert.assertEquals(flowSpec.getConfig().getBoolean(RUN_IMMEDIATELY_KEY), true); Assert.assertEquals(flowSpec.getConfig().getString("param1"),"value1"); Assert.assertEquals(flowSpec.getConfig().getString("param2"),"value1-123"); Assert.assertEquals(flowSpec.getConfig().getString("param3"), "a:b:c*.d"); Assert.assertEquals(flowSpec.getTemplateURIs().get().size(), 1); Assert.assertTrue(flowSpec.getTemplateURIs().get().contains(new URI(TEST_TEMPLATE_URI)));
JobSpec jobSpec; JobSpec.Builder jobSpecBuilder = JobSpec.builder(jobSpecURIGenerator(flowSpec)) .withConfig(flowSpec.getConfig()) .withDescription(flowSpec.getDescription()) .withVersion(flowSpec.getVersion()); if (flowSpec.getConfig().hasPath(ConfigurationKeys.FLOW_NAME_KEY)) { jobSpec.setConfig(jobSpec.getConfig() .withValue(ConfigurationKeys.JOB_NAME_KEY, flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_NAME_KEY))); if (flowSpec.getConfig().hasPath(ConfigurationKeys.FLOW_GROUP_KEY)) { jobSpec.setConfig(jobSpec.getConfig() .withValue(ConfigurationKeys.JOB_GROUP_KEY, flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_GROUP_KEY)));
this.flowGraph = flowGraph; this.flowSpec = flowSpec; this.flowConfig = flowSpec.getConfig(); this.flowExecutionId = FlowUtils.getOrCreateFlowExecutionId(flowSpec);
JobSpec jobSpec; JobSpec.Builder jobSpecBuilder = JobSpec.builder(jobSpecURIGenerator(flowSpec, sourceNode, targetNode)) .withConfig(flowSpec.getConfig()) .withDescription(flowSpec.getDescription()) .withVersion(flowSpec.getVersion()); if (flowSpec.getConfig().hasPath(ConfigurationKeys.FLOW_NAME_KEY)) { jobSpec.setConfig(jobSpec.getConfig() .withValue(ConfigurationKeys.JOB_NAME_KEY, ConfigValueFactory.fromAnyRef( flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_NAME_KEY).unwrapped().toString() + "-" + sourceNode.getNodeName() + "-" + targetNode.getNodeName()))); if (flowSpec.getConfig().hasPath(ConfigurationKeys.FLOW_GROUP_KEY)) { jobSpec.setConfig(jobSpec.getConfig() .withValue(ConfigurationKeys.JOB_GROUP_KEY, flowSpec.getConfig().getValue(ConfigurationKeys.FLOW_GROUP_KEY)));
Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow"); Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup"); Assert.assertEquals(spec.getConfig().getString("param1"), "value1"); Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow2"); Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup"); Assert.assertEquals(spec.getConfig().getString("param1"), "value2"); Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow2"); Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup"); Assert.assertEquals(spec.getConfig().getString("param1"), "value4"); Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY), "testFlow3"); Assert.assertEquals(spec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY), "testGroup"); Assert.assertEquals(spec.getConfig().getString("param1"), "value5");
Config flowConfig = flowSpec.getConfig();
static Map<String, String> getFlowMetadata(FlowSpec flowSpec) { Map<String, String> metadata = Maps.newHashMap(); metadata.put(TimingEvent.FlowEventConstants.FLOW_NAME_FIELD, flowSpec.getConfig().getString(ConfigurationKeys.FLOW_NAME_KEY)); metadata.put(TimingEvent.FlowEventConstants.FLOW_GROUP_FIELD, flowSpec.getConfig().getString(ConfigurationKeys.FLOW_GROUP_KEY)); Config flowConfig = flowSpec.getConfig(); if (flowConfig.hasPath(ConfigurationKeys.FLOW_EXECUTION_ID_KEY)) { metadata.put(TimingEvent.FlowEventConstants.FLOW_EXECUTION_ID_FIELD, flowConfig.getString(ConfigurationKeys.FLOW_EXECUTION_ID_KEY)); } return metadata; }
/** * A FlowSpec contains a FlowExecutionId if it is a runOnce flow. * Refer {@link FlowConfigResourceLocalHandler#createFlowSpecForConfig} for details. * @param spec flow spec * @return flow execution id */ public static long getOrCreateFlowExecutionId(FlowSpec spec) { long flowExecutionId; if (spec.getConfig().hasPath(ConfigurationKeys.FLOW_EXECUTION_ID_KEY)) { flowExecutionId = spec.getConfig().getLong(ConfigurationKeys.FLOW_EXECUTION_ID_KEY); } else { flowExecutionId = System.currentTimeMillis(); } return flowExecutionId; } }