public static FlowConfig deserializeFlowConfig(String serialized) throws IOException { Properties properties = PropertiesUtils.deserialize(serialized); FlowConfig flowConfig = new FlowConfig().setId(new FlowId() .setFlowName(properties.getProperty(FLOWCONFIG_ID_NAME)) .setFlowGroup(properties.getProperty(FLOWCONFIG_ID_GROUP))); if (properties.containsKey(FLOWCONFIG_SCHEDULE_CRON)) { flowConfig.setSchedule(new Schedule() .setCronSchedule(properties.getProperty(FLOWCONFIG_SCHEDULE_CRON)) .setRunImmediately(Boolean.valueOf(properties.getProperty(FLOWCONFIG_SCHEDULE_RUN_IMMEDIATELY)))); } if (properties.containsKey(FLOWCONFIG_TEMPLATEURIS)) { flowConfig.setTemplateUris(properties.getProperty(FLOWCONFIG_TEMPLATEURIS)); } properties.remove(FLOWCONFIG_ID_NAME); properties.remove(FLOWCONFIG_ID_GROUP); properties.remove(FLOWCONFIG_SCHEDULE_CRON); properties.remove(FLOWCONFIG_SCHEDULE_RUN_IMMEDIATELY); properties.remove(FLOWCONFIG_TEMPLATEURIS); flowConfig.setProperties(new StringMap(Maps.fromProperties(properties))); return flowConfig; } }
public static String serializeFlowConfig(FlowConfig flowConfig) throws IOException { Properties properties = ConfigUtils.configToProperties(ConfigFactory.parseMap(flowConfig.getProperties())); properties.setProperty(FLOWCONFIG_ID_NAME, flowConfig.getId().getFlowName()); properties.setProperty(FLOWCONFIG_ID_GROUP, flowConfig.getId().getFlowGroup()); if (flowConfig.hasSchedule()) { properties.setProperty(FLOWCONFIG_SCHEDULE_CRON, flowConfig.getSchedule().getCronSchedule()); properties.setProperty(FLOWCONFIG_SCHEDULE_RUN_IMMEDIATELY, Boolean.toString(flowConfig.getSchedule().isRunImmediately())); } if (flowConfig.hasTemplateUris()) { properties.setProperty(FLOWCONFIG_TEMPLATEURIS, flowConfig.getTemplateUris()); } return PropertiesUtils.serialize(properties); }
@Test (dependsOnMethods = "testCreate") public void testCreateAgain() throws Exception { Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1"); flowProperties.put(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY, TEST_SOURCE_NAME); flowProperties.put(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY, TEST_SINK_NAME); FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE)) .setProperties(new StringMap(flowProperties)); try { this.flowConfigClient.createFlowConfig(flowConfig); } catch (RestLiResponseException e) { Assert.fail("Create Again should pass without complaining that the spec already exists."); } }
@Test (dependsOnMethods = "testGet") public void testUpdate() throws Exception { logger.info("+++++++++++++++++++ testUpdate START"); // Update on one node and retrieve from another FlowId flowId = new FlowId().setFlowGroup(TEST_GROUP_NAME_1).setFlowName(TEST_FLOW_NAME_1); Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1b"); flowProperties.put("param2", "value2b"); flowProperties.put(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY, TEST_SOURCE_NAME); flowProperties.put(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY, TEST_SINK_NAME); FlowConfig flowConfig = new FlowConfig() .setId(new FlowId().setFlowGroup(TEST_GROUP_NAME_1).setFlowName(TEST_FLOW_NAME_1)) .setTemplateUris(TEST_TEMPLATE_URI_1).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE_1)) .setProperties(new StringMap(flowProperties)); this.node1FlowConfigClient.updateFlowConfig(flowConfig); FlowConfig retrievedFlowConfig = this.node2FlowConfigClient.getFlowConfig(flowId); Assert.assertEquals(retrievedFlowConfig.getId().getFlowGroup(), TEST_GROUP_NAME_1); Assert.assertEquals(retrievedFlowConfig.getId().getFlowName(), TEST_FLOW_NAME_1); Assert.assertEquals(retrievedFlowConfig.getSchedule().getCronSchedule(), TEST_SCHEDULE_1); Assert.assertEquals(retrievedFlowConfig.getTemplateUris(), TEST_TEMPLATE_URI_1); Assert.assertFalse(retrievedFlowConfig.getSchedule().isRunImmediately()); Assert.assertEquals(retrievedFlowConfig.getProperties().get("param1"), "value1b"); Assert.assertEquals(retrievedFlowConfig.getProperties().get("param2"), "value2b"); logger.info("+++++++++++++++++++ testUpdate END"); }
@Test (dependsOnMethods = "testGet") public void testUpdate() throws Exception { FlowId flowId = new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME); Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1b"); flowProperties.put("param2", "value2b"); flowProperties.put(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY, TEST_SOURCE_NAME); flowProperties.put(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY, TEST_SINK_NAME); FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE)) .setProperties(new StringMap(flowProperties)); this.flowConfigClient.updateFlowConfig(flowConfig); FlowConfig retrievedFlowConfig = this.flowConfigClient.getFlowConfig(flowId); Assert.assertEquals(retrievedFlowConfig.getId().getFlowGroup(), TEST_GROUP_NAME); Assert.assertEquals(retrievedFlowConfig.getId().getFlowName(), TEST_FLOW_NAME); Assert.assertEquals(retrievedFlowConfig.getSchedule().getCronSchedule(), TEST_SCHEDULE); Assert.assertEquals(retrievedFlowConfig.getTemplateUris(), TEST_TEMPLATE_URI); // Add this asssert when getFlowSpec() is changed to return the raw flow spec //Assert.assertEquals(flowConfig.getProperties().size(), 2); Assert.assertEquals(retrievedFlowConfig.getProperties().get("param1"), "value1b"); Assert.assertEquals(retrievedFlowConfig.getProperties().get("param2"), "value2b"); }
/** * Existence checker for runImmediately * * @see Schedule.Fields#runImmediately */ public boolean hasRunImmediately() { return contains(FIELD_RunImmediately); }
@Test (dependsOnMethods = "testCreate") public void testCreateAgain() throws Exception { Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1"); FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE)) .setProperties(new StringMap(flowProperties)); try { _client.createFlowConfig(flowConfig); } catch (RestLiResponseException e) { Assert.fail("Create Again should pass without complaining that the spec already exists."); } }
@Test (dependsOnMethods = "testGet") public void testUpdate() throws Exception { FlowId flowId = new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME); Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1b"); flowProperties.put("param2", "value2b"); FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE)) .setProperties(new StringMap(flowProperties)); _client.updateFlowConfig(flowConfig); FlowConfig retrievedFlowConfig = _client.getFlowConfig(flowId); Assert.assertEquals(retrievedFlowConfig.getId().getFlowGroup(), TEST_GROUP_NAME); Assert.assertEquals(retrievedFlowConfig.getId().getFlowName(), TEST_FLOW_NAME); Assert.assertEquals(retrievedFlowConfig.getSchedule().getCronSchedule(), TEST_SCHEDULE ); Assert.assertEquals(retrievedFlowConfig.getTemplateUris(), TEST_TEMPLATE_URI); // Add this asssert when getFlowSpec() is changed to return the raw flow spec //Assert.assertEquals(flowConfig.getProperties().size(), 2); Assert.assertEquals(retrievedFlowConfig.getProperties().get("param1"), "value1b"); Assert.assertEquals(retrievedFlowConfig.getProperties().get("param2"), "value2b"); }
/** * Existence checker for cronSchedule * * @see Schedule.Fields#cronSchedule */ public boolean hasCronSchedule() { return contains(FIELD_CronSchedule); }
@SuppressWarnings("DLS_DEAD_LOCAL_STORE") private static void testGobblinService(GobblinServiceManager gobblinServiceManager) { FlowConfigClient client = new FlowConfigClient(String.format("http://localhost:%s/", gobblinServiceManager.restliServer.getPort())); Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1"); final String TEST_GROUP_NAME = "testGroup1"; final String TEST_FLOW_NAME = "testFlow1"; final String TEST_SCHEDULE = "0 1/0 * ? * *"; final String TEST_TEMPLATE_URI = "FS:///templates/test.template"; FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE). setRunImmediately(true)) .setProperties(new StringMap(flowProperties)); try { client.createFlowConfig(flowConfig); } catch (RemoteInvocationException e) { throw new RuntimeException(e); } } }
@Test public void testBadUpdate() throws Exception { Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1b"); flowProperties.put("param2", "value2b"); FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_DUMMY_GROUP_NAME) .setFlowName(TEST_DUMMY_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE)) .setProperties(new StringMap(flowProperties)); try { _client.updateFlowConfig(flowConfig); } catch (RestLiResponseException e) { Assert.fail("Bad update should pass without complaining that the spec does not exists."); } }
@Test (dependsOnMethods = "testCreateAgain") public void testGet() throws Exception { FlowId flowId = new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME); FlowConfig flowConfig = _client.getFlowConfig(flowId); Assert.assertEquals(flowConfig.getId().getFlowGroup(), TEST_GROUP_NAME); Assert.assertEquals(flowConfig.getId().getFlowName(), TEST_FLOW_NAME); Assert.assertEquals(flowConfig.getSchedule().getCronSchedule(), TEST_SCHEDULE ); Assert.assertEquals(flowConfig.getTemplateUris(), TEST_TEMPLATE_URI); Assert.assertFalse(flowConfig.getSchedule().isRunImmediately()); // Add this asssert back when getFlowSpec() is changed to return the raw flow spec //Assert.assertEquals(flowConfig.getProperties().size(), 1); Assert.assertEquals(flowConfig.getProperties().get("param1"), "value1"); }
schedule = new Schedule(); schedule.setCronSchedule(flowProps.getProperty(ConfigurationKeys.JOB_SCHEDULE_KEY)); schedule.setRunImmediately(Boolean.valueOf(flowProps.getProperty(ConfigurationKeys.FLOW_RUN_IMMEDIATELY)));
@Test (dependsOnMethods = "testBadDelete") public void testBadUpdate() throws Exception { logger.info("+++++++++++++++++++ testBadUpdate START"); Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1b"); flowProperties.put("param2", "value2b"); FlowConfig flowConfig = new FlowConfig() .setId(new FlowId().setFlowGroup(TEST_DUMMY_GROUP_NAME_1).setFlowName(TEST_DUMMY_FLOW_NAME_1)) .setTemplateUris(TEST_TEMPLATE_URI_1).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE_1)) .setProperties(new StringMap(flowProperties)); try { this.node1FlowConfigClient.updateFlowConfig(flowConfig); } catch (RestLiResponseException e) { Assert.fail("Bad update should pass without complaining that the spec does not exists."); } try { this.node2FlowConfigClient.updateFlowConfig(flowConfig); } catch (RestLiResponseException e) { Assert.fail("Bad update should pass without complaining that the spec does not exists."); } logger.info("+++++++++++++++++++ testBadUpdate END"); }
@Test (dependsOnMethods = "testCreateAgain") public void testGet() throws Exception { FlowId flowId = new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME); FlowConfig flowConfig = this.flowConfigClient.getFlowConfig(flowId); Assert.assertEquals(flowConfig.getId().getFlowGroup(), TEST_GROUP_NAME); Assert.assertEquals(flowConfig.getId().getFlowName(), TEST_FLOW_NAME); Assert.assertEquals(flowConfig.getSchedule().getCronSchedule(), TEST_SCHEDULE); Assert.assertEquals(flowConfig.getTemplateUris(), TEST_TEMPLATE_URI); Assert.assertFalse(flowConfig.getSchedule().isRunImmediately()); // Add this assert back when getFlowSpec() is changed to return the raw flow spec //Assert.assertEquals(flowConfig.getProperties().size(), 1); Assert.assertEquals(flowConfig.getProperties().get("param1"), "value1"); }
@Test public void testCreate() throws Exception { Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1"); flowProperties.put(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY, TEST_SOURCE_NAME); flowProperties.put(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY, TEST_SINK_NAME); FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE).setRunImmediately(true)) .setProperties(new StringMap(flowProperties)); this.flowConfigClient.createFlowConfig(flowConfig); Assert.assertTrue(this.gobblinServiceManager.flowCatalog.getSpecs().size() == 1, "Flow that was created is not " + "reflecting in FlowCatalog"); }
@Test public void testBadUpdate() throws Exception { Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1b"); flowProperties.put("param2", "value2b"); FlowConfig flowConfig = new FlowConfig() .setId(new FlowId().setFlowGroup(TEST_DUMMY_GROUP_NAME).setFlowName(TEST_DUMMY_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE)) .setProperties(new StringMap(flowProperties)); try { this.flowConfigClient.updateFlowConfig(flowConfig); } catch (RestLiResponseException e) { Assert.fail("Bad update should pass without complaining that the spec does not exists."); } cleanUpDir(FLOW_SPEC_STORE_DIR); } }
@Test (dependsOnMethods = "testCreateAgain") public void testGet() throws Exception { logger.info("+++++++++++++++++++ testGet START"); FlowId flowId1 = new FlowId().setFlowGroup(TEST_GROUP_NAME_1).setFlowName(TEST_FLOW_NAME_1); FlowConfig flowConfig1 = this.node1FlowConfigClient.getFlowConfig(flowId1); Assert.assertEquals(flowConfig1.getId().getFlowGroup(), TEST_GROUP_NAME_1); Assert.assertEquals(flowConfig1.getId().getFlowName(), TEST_FLOW_NAME_1); Assert.assertEquals(flowConfig1.getSchedule().getCronSchedule(), TEST_SCHEDULE_1); Assert.assertEquals(flowConfig1.getTemplateUris(), TEST_TEMPLATE_URI_1); Assert.assertTrue(flowConfig1.getSchedule().isRunImmediately()); Assert.assertEquals(flowConfig1.getProperties().get("param1"), "value1"); flowConfig1 = this.node2FlowConfigClient.getFlowConfig(flowId1); Assert.assertEquals(flowConfig1.getId().getFlowGroup(), TEST_GROUP_NAME_1); Assert.assertEquals(flowConfig1.getId().getFlowName(), TEST_FLOW_NAME_1); Assert.assertEquals(flowConfig1.getSchedule().getCronSchedule(), TEST_SCHEDULE_1); Assert.assertEquals(flowConfig1.getTemplateUris(), TEST_TEMPLATE_URI_1); Assert.assertTrue(flowConfig1.getSchedule().isRunImmediately()); Assert.assertEquals(flowConfig1.getProperties().get("param1"), "value1"); logger.info("+++++++++++++++++++ testGet END"); }
@Test public void testCreate() throws Exception { Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1"); FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE). setRunImmediately(true)) .setProperties(new StringMap(flowProperties)); _client.createFlowConfig(flowConfig); }
public void testFlowConfigWithoutTemplateUri() { FlowConfig flowConfig = new FlowConfig().setId(new FlowId() .setFlowName("SN_CRMSYNC") .setFlowGroup("DYNAMICS-USER-123456789")); flowConfig.setSchedule(new Schedule() .setCronSchedule("0 58 2/12 ? * * *")); Properties properties = new Properties(); properties.put("gobblin.flow.sourceIdentifier", "dynamicsCrm"); properties.put("gobblin.flow.destinationIdentifier", "espresso"); flowConfig.setProperties(new StringMap(Maps.fromProperties(properties))); try { FlowConfigResourceLocalHandler.createFlowSpecForConfig(flowConfig); Assert.fail("Should not get to here"); } catch (RequiredFieldNotPresentException e) { Assert.assertTrue(true, "templateUri cannot be empty"); } testSerDer(flowConfig); } }