public static FlowConfig deserializeFlowConfig(String serialized) throws IOException { Properties properties = PropertiesUtils.deserialize(serialized); FlowConfig flowConfig = new FlowConfig().setId(new FlowId() .setFlowName(properties.getProperty(FLOWCONFIG_ID_NAME)) .setFlowGroup(properties.getProperty(FLOWCONFIG_ID_GROUP))); if (properties.containsKey(FLOWCONFIG_SCHEDULE_CRON)) { flowConfig.setSchedule(new Schedule() .setCronSchedule(properties.getProperty(FLOWCONFIG_SCHEDULE_CRON)) .setRunImmediately(Boolean.valueOf(properties.getProperty(FLOWCONFIG_SCHEDULE_RUN_IMMEDIATELY)))); } if (properties.containsKey(FLOWCONFIG_TEMPLATEURIS)) { flowConfig.setTemplateUris(properties.getProperty(FLOWCONFIG_TEMPLATEURIS)); } properties.remove(FLOWCONFIG_ID_NAME); properties.remove(FLOWCONFIG_ID_GROUP); properties.remove(FLOWCONFIG_SCHEDULE_CRON); properties.remove(FLOWCONFIG_SCHEDULE_RUN_IMMEDIATELY); properties.remove(FLOWCONFIG_TEMPLATEURIS); flowConfig.setProperties(new StringMap(Maps.fromProperties(properties))); return flowConfig; } }
schedule.setRunImmediately(Boolean.valueOf(flowProps.getProperty(ConfigurationKeys.FLOW_RUN_IMMEDIATELY)));
@SuppressWarnings("DLS_DEAD_LOCAL_STORE") private static void testGobblinService(GobblinServiceManager gobblinServiceManager) { FlowConfigClient client = new FlowConfigClient(String.format("http://localhost:%s/", gobblinServiceManager.restliServer.getPort())); Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1"); final String TEST_GROUP_NAME = "testGroup1"; final String TEST_FLOW_NAME = "testFlow1"; final String TEST_SCHEDULE = "0 1/0 * ? * *"; final String TEST_TEMPLATE_URI = "FS:///templates/test.template"; FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE). setRunImmediately(true)) .setProperties(new StringMap(flowProperties)); try { client.createFlowConfig(flowConfig); } catch (RemoteInvocationException e) { throw new RuntimeException(e); } } }
@Test public void testCreate() throws Exception { Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1"); flowProperties.put(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY, TEST_SOURCE_NAME); flowProperties.put(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY, TEST_SINK_NAME); FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE).setRunImmediately(true)) .setProperties(new StringMap(flowProperties)); this.flowConfigClient.createFlowConfig(flowConfig); Assert.assertTrue(this.gobblinServiceManager.flowCatalog.getSpecs().size() == 1, "Flow that was created is not " + "reflecting in FlowCatalog"); }
setRunImmediately(true)) .setProperties(new StringMap(flowProperties)); flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE). setRunImmediately(true)) .setProperties(new StringMap(flowProperties)); flowSpec = FlowConfigResourceLocalHandler.createFlowSpecForConfig(flowConfig);
@Test public void testCreate() throws Exception { Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1"); FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE). setRunImmediately(true)) .setProperties(new StringMap(flowProperties)); _client.createFlowConfig(flowConfig); }
@Test public void testCreateBadSchedule() throws Exception { Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1"); FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule("bad schedule"). setRunImmediately(true)) .setProperties(new StringMap(flowProperties)); try { _client.createFlowConfig(flowConfig); } catch (RestLiResponseException e) { Assert.assertEquals(e.getStatus(), HttpStatus.S_422_UNPROCESSABLE_ENTITY.getCode()); return; } Assert.fail("Get should have gotten a 422 error"); }
@Test public void testCreateBadTemplateUri() throws Exception { Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1"); FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris("FILE://bad/uri").setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE). setRunImmediately(true)) .setProperties(new StringMap(flowProperties)); try { _client.createFlowConfig(flowConfig); } catch (RestLiResponseException e) { Assert.assertEquals(e.getStatus(), HttpStatus.S_422_UNPROCESSABLE_ENTITY.getCode()); return; } Assert.fail("Get should have gotten a 422 error"); }
public void testFullFlowConfig() { FlowConfig flowConfig = new FlowConfig().setId(new FlowId() .setFlowName("SN_CRMSYNC") .setFlowGroup("DYNAMICS-USER-123456789")); flowConfig.setSchedule(new Schedule() .setCronSchedule("0 58 2/12 ? * * *") .setRunImmediately(Boolean.valueOf("true"))); flowConfig.setTemplateUris("FS:///my.template"); Properties properties = new Properties(); properties.put("gobblin.flow.sourceIdentifier", "dynamicsCrm"); properties.put("gobblin.flow.destinationIdentifier", "espresso"); flowConfig.setProperties(new StringMap(Maps.fromProperties(properties))); testFlowSpec(flowConfig); testSerDer(flowConfig); }
.setId(new FlowId().setFlowGroup(TEST_GROUP_NAME_1).setFlowName(TEST_FLOW_NAME_1)) .setTemplateUris(TEST_TEMPLATE_URI_1).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE_1). setRunImmediately(true)) .setProperties(new StringMap(flowProperties)); FlowConfig flowConfig2 = new FlowConfig() .setId(new FlowId().setFlowGroup(TEST_GROUP_NAME_2).setFlowName(TEST_FLOW_NAME_2)) .setTemplateUris(TEST_TEMPLATE_URI_2).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE_2). setRunImmediately(true)) .setProperties(new StringMap(flowProperties));
@Test public void testCheckFlowExecutionId() throws Exception { Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1"); FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setProperties(new StringMap(flowProperties)); FlowStatusId flowStatusId =_client.createFlowConfig(flowConfig); Assert.assertEquals(TEST_GROUP_NAME, flowStatusId.getFlowGroup()); Assert.assertEquals(TEST_FLOW_NAME, flowStatusId.getFlowName()); Assert.assertTrue(flowStatusId.getFlowExecutionId() != -1); flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setProperties(new StringMap(flowProperties)) .setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE).setRunImmediately(true)); Assert.assertEquals(_client.createFlowConfig(flowConfig).getFlowExecutionId().longValue(), -1L); }
@Test (dependsOnMethods = "testCreate") public void testCreateAgain() throws Exception { logger.info("+++++++++++++++++++ testCreateAgain START"); Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1"); flowProperties.put(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY, TEST_SOURCE_NAME); flowProperties.put(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY, TEST_SINK_NAME); FlowConfig flowConfig1 = new FlowConfig() .setId(new FlowId().setFlowGroup(TEST_GROUP_NAME_1).setFlowName(TEST_FLOW_NAME_1)) .setTemplateUris(TEST_TEMPLATE_URI_1).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE_1). setRunImmediately(true)) .setProperties(new StringMap(flowProperties)); FlowConfig flowConfig2 = new FlowConfig() .setId(new FlowId().setFlowGroup(TEST_GROUP_NAME_2).setFlowName(TEST_FLOW_NAME_2)) .setTemplateUris(TEST_TEMPLATE_URI_2).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE_2). setRunImmediately(true)) .setProperties(new StringMap(flowProperties)); // Try create on both nodes try { this.node1FlowConfigClient.createFlowConfig(flowConfig1); } catch (RestLiResponseException e) { Assert.fail("Create Again should pass without complaining that the spec already exists."); } try { this.node2FlowConfigClient.createFlowConfig(flowConfig2); } catch (RestLiResponseException e) { Assert.fail("Create Again should pass without complaining that the spec already exists."); } logger.info("+++++++++++++++++++ testCreateAgain END"); }
public static FlowConfig deserializeFlowConfig(String serialized) throws IOException { Properties properties = PropertiesUtils.deserialize(serialized); FlowConfig flowConfig = new FlowConfig().setId(new FlowId() .setFlowName(properties.getProperty(FLOWCONFIG_ID_NAME)) .setFlowGroup(properties.getProperty(FLOWCONFIG_ID_GROUP))); if (properties.containsKey(FLOWCONFIG_SCHEDULE_CRON)) { flowConfig.setSchedule(new Schedule() .setCronSchedule(properties.getProperty(FLOWCONFIG_SCHEDULE_CRON)) .setRunImmediately(Boolean.valueOf(properties.getProperty(FLOWCONFIG_SCHEDULE_RUN_IMMEDIATELY)))); } if (properties.containsKey(FLOWCONFIG_TEMPLATEURIS)) { flowConfig.setTemplateUris(properties.getProperty(FLOWCONFIG_TEMPLATEURIS)); } properties.remove(FLOWCONFIG_ID_NAME); properties.remove(FLOWCONFIG_ID_GROUP); properties.remove(FLOWCONFIG_SCHEDULE_CRON); properties.remove(FLOWCONFIG_SCHEDULE_RUN_IMMEDIATELY); properties.remove(FLOWCONFIG_TEMPLATEURIS); flowConfig.setProperties(new StringMap(Maps.fromProperties(properties))); return flowConfig; } }
@SuppressWarnings("DLS_DEAD_LOCAL_STORE") private static void testGobblinService(GobblinServiceManager gobblinServiceManager) { FlowConfigClient client = new FlowConfigClient(String.format("http://localhost:%s/", gobblinServiceManager.restliServer.getPort())); Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1"); final String TEST_GROUP_NAME = "testGroup1"; final String TEST_FLOW_NAME = "testFlow1"; final String TEST_SCHEDULE = "0 1/0 * ? * *"; final String TEST_TEMPLATE_URI = "FS:///templates/test.template"; FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE). setRunImmediately(true)) .setProperties(new StringMap(flowProperties)); try { client.createFlowConfig(flowConfig); } catch (RemoteInvocationException e) { throw new RuntimeException(e); } } }