@Test (dependsOnMethods = "testGet") public void testUpdate() throws Exception { FlowId flowId = new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME); Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1b"); flowProperties.put("param2", "value2b"); flowProperties.put(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY, TEST_SOURCE_NAME); flowProperties.put(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY, TEST_SINK_NAME); FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE)) .setProperties(new StringMap(flowProperties)); this.flowConfigClient.updateFlowConfig(flowConfig); FlowConfig retrievedFlowConfig = this.flowConfigClient.getFlowConfig(flowId); Assert.assertEquals(retrievedFlowConfig.getId().getFlowGroup(), TEST_GROUP_NAME); Assert.assertEquals(retrievedFlowConfig.getId().getFlowName(), TEST_FLOW_NAME); Assert.assertEquals(retrievedFlowConfig.getSchedule().getCronSchedule(), TEST_SCHEDULE); Assert.assertEquals(retrievedFlowConfig.getTemplateUris(), TEST_TEMPLATE_URI); // Add this asssert when getFlowSpec() is changed to return the raw flow spec //Assert.assertEquals(flowConfig.getProperties().size(), 2); Assert.assertEquals(retrievedFlowConfig.getProperties().get("param1"), "value1b"); Assert.assertEquals(retrievedFlowConfig.getProperties().get("param2"), "value2b"); }
public static String serializeFlowConfig(FlowConfig flowConfig) throws IOException { Properties properties = ConfigUtils.configToProperties(ConfigFactory.parseMap(flowConfig.getProperties())); properties.setProperty(FLOWCONFIG_ID_NAME, flowConfig.getId().getFlowName()); properties.setProperty(FLOWCONFIG_ID_GROUP, flowConfig.getId().getFlowGroup()); if (flowConfig.hasSchedule()) { properties.setProperty(FLOWCONFIG_SCHEDULE_CRON, flowConfig.getSchedule().getCronSchedule()); properties.setProperty(FLOWCONFIG_SCHEDULE_RUN_IMMEDIATELY, Boolean.toString(flowConfig.getSchedule().isRunImmediately())); } if (flowConfig.hasTemplateUris()) { properties.setProperty(FLOWCONFIG_TEMPLATEURIS, flowConfig.getTemplateUris()); } return PropertiesUtils.serialize(properties); }
public static FlowConfig deserializeFlowConfig(String serialized) throws IOException { Properties properties = PropertiesUtils.deserialize(serialized); FlowConfig flowConfig = new FlowConfig().setId(new FlowId() .setFlowName(properties.getProperty(FLOWCONFIG_ID_NAME)) .setFlowGroup(properties.getProperty(FLOWCONFIG_ID_GROUP))); if (properties.containsKey(FLOWCONFIG_SCHEDULE_CRON)) { flowConfig.setSchedule(new Schedule() .setCronSchedule(properties.getProperty(FLOWCONFIG_SCHEDULE_CRON)) .setRunImmediately(Boolean.valueOf(properties.getProperty(FLOWCONFIG_SCHEDULE_RUN_IMMEDIATELY)))); } if (properties.containsKey(FLOWCONFIG_TEMPLATEURIS)) { flowConfig.setTemplateUris(properties.getProperty(FLOWCONFIG_TEMPLATEURIS)); } properties.remove(FLOWCONFIG_ID_NAME); properties.remove(FLOWCONFIG_ID_GROUP); properties.remove(FLOWCONFIG_SCHEDULE_CRON); properties.remove(FLOWCONFIG_SCHEDULE_RUN_IMMEDIATELY); properties.remove(FLOWCONFIG_TEMPLATEURIS); flowConfig.setProperties(new StringMap(Maps.fromProperties(properties))); return flowConfig; } }
if (f1.equals(f2)) { return true; Assert.assertTrue(f1.hasId() == f2.hasId()); Assert.assertTrue(f1.getId().equals(f2.getId())); Assert.assertTrue(f1.hasSchedule() == f2.hasSchedule()); if (f1.hasSchedule()) { Schedule s1 = f1.getSchedule(); Schedule s2 = f2.getSchedule(); Assert.assertTrue(s1.getCronSchedule().equals(s2.getCronSchedule())); Assert.assertTrue(s1.isRunImmediately().equals(s2.isRunImmediately())); Assert.assertTrue(f1.hasTemplateUris() == f2.hasTemplateUris()); if (f1.hasTemplateUris()) { Assert.assertTrue(f1.getTemplateUris().equals(f2.getTemplateUris())); Assert.assertTrue(f1.hasProperties() == f2.hasProperties()); if (f1.hasProperties()) { Assert.assertTrue(f1.getProperties().equals(f2.getProperties()));
/** * Create a flow configuration * @param flowConfig flow configuration attributes * @throws RemoteInvocationException */ public void createFlowConfig(FlowConfig flowConfig) throws RemoteInvocationException { LOG.debug("createFlowConfig with groupName " + flowConfig.getId().getFlowGroup() + " flowName " + flowConfig.getId().getFlowName()); CreateIdRequest<ComplexResourceKey<FlowId, EmptyRecord>, FlowConfig> request = _flowconfigsRequestBuilders.create().input(flowConfig).build(); ResponseFuture<IdResponse<ComplexResourceKey<FlowId, EmptyRecord>>> flowConfigResponseFuture = _restClient.get().sendRequest(request); flowConfigResponseFuture.getResponse(); }
/** * Add flowConfig locally and trigger all listeners iff @param triggerListener is set to true */ public CreateResponse createFlowConfig(FlowConfig flowConfig, boolean triggerListener) throws FlowConfigLoggedException { log.info("[GAAS-REST] Create called with flowGroup " + flowConfig.getId().getFlowGroup() + " flowName " + flowConfig.getId().getFlowName()); FlowSpec flowSpec = createFlowSpecForConfig(flowConfig); // Existence of a flow spec in the flow catalog implies that the flow is currently running. // If the new flow spec has a schedule we should allow submission of the new flow to accept the new schedule. // However, if the new flow spec does not have a schedule, we should allow submission only if it is not running. if (!flowConfig.hasSchedule() && this.flowCatalog.exists(flowSpec.getUri())) { return new CreateResponse(new ComplexResourceKey<>(flowConfig.getId(), new EmptyRecord()), HttpStatus.S_409_CONFLICT); } else { this.flowCatalog.put(flowSpec, triggerListener); return new CreateResponse(new ComplexResourceKey<>(flowConfig.getId(), new EmptyRecord()), HttpStatus.S_201_CREATED); } }
/** * Create a flow configuration that the service will forward to execution instances for execution * @param flowConfig flow configuration * @return {@link CreateResponse} */ @Override public CreateResponse create(FlowConfig flowConfig) { List<ServiceRequester> requestorList = this.requesterService.findRequesters(this); try { String serialized = this.requesterService.serialize(requestorList); flowConfig.getProperties().put(RequesterService.REQUESTER_LIST, serialized); LOG.info("Rest requester list is " + serialized); } catch (IOException e) { throw new FlowConfigLoggedException(HttpStatus.S_401_UNAUTHORIZED, "cannot get who is the requester", e); } return this.flowConfigsResourceHandler.createFlowConfig(flowConfig); }
@SuppressWarnings("DLS_DEAD_LOCAL_STORE") private static void testGobblinService(GobblinServiceManager gobblinServiceManager) { FlowConfigClient client = new FlowConfigClient(String.format("http://localhost:%s/", gobblinServiceManager.restliServer.getPort())); Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1"); final String TEST_GROUP_NAME = "testGroup1"; final String TEST_FLOW_NAME = "testFlow1"; final String TEST_SCHEDULE = "0 1/0 * ? * *"; final String TEST_TEMPLATE_URI = "FS:///templates/test.template"; FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE). setRunImmediately(true)) .setProperties(new StringMap(flowProperties)); try { client.createFlowConfig(flowConfig); } catch (RemoteInvocationException e) { throw new RuntimeException(e); } } }
/** * Update flowConfig locally and trigger all listeners iff @param triggerListener is set to true */ public UpdateResponse updateFlowConfig(FlowId flowId, FlowConfig flowConfig, boolean triggerListener) { log.info("[GAAS-REST] Update called with flowGroup {} flowName {}", flowId.getFlowGroup(), flowId.getFlowName()); if (!flowId.getFlowGroup().equals(flowConfig.getId().getFlowGroup()) || !flowId.getFlowName().equals(flowConfig.getId().getFlowName())) { throw new FlowConfigLoggedException(HttpStatus.S_400_BAD_REQUEST, "flowName and flowGroup cannot be changed in update", null); } this.flowCatalog.put(createFlowSpecForConfig(flowConfig), triggerListener); return new UpdateResponse(HttpStatus.S_200_OK); }
"/" + flowId.getFlowGroup() + "/" + flowId.getFlowName(), null, null); FlowSpec spec = (FlowSpec) flowCatalog.getSpec(flowUri); FlowConfig flowConfig = new FlowConfig(); Properties flowProps = spec.getConfigAsProperties(); Schedule schedule = null; flowConfig.setTemplateUris(flowProps.getProperty(ConfigurationKeys.JOB_TEMPLATE_PATH)); } else if (spec.getTemplateURIs().isPresent()) { flowConfig.setTemplateUris(StringUtils.join(spec.getTemplateURIs().get(), ",")); } else { flowConfig.setTemplateUris("NA"); flowConfig.setSchedule(schedule); flowPropsAsStringMap.putAll(Maps.fromProperties(flowProps)); return flowConfig.setId(new FlowId().setFlowGroup(flowId.getFlowGroup()).setFlowName(flowId.getFlowName())) .setProperties(flowPropsAsStringMap); } catch (URISyntaxException e) { throw new FlowConfigLoggedException(HttpStatus.S_400_BAD_REQUEST, "bad URI " + flowId.getFlowName(), e);
@Test (dependsOnMethods = "testGet") public void testUpdate() throws Exception { FlowId flowId = new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME); Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1b"); flowProperties.put("param2", "value2b"); FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE)) .setProperties(new StringMap(flowProperties)); _client.updateFlowConfig(flowConfig); FlowConfig retrievedFlowConfig = _client.getFlowConfig(flowId); Assert.assertEquals(retrievedFlowConfig.getId().getFlowGroup(), TEST_GROUP_NAME); Assert.assertEquals(retrievedFlowConfig.getId().getFlowName(), TEST_FLOW_NAME); Assert.assertEquals(retrievedFlowConfig.getSchedule().getCronSchedule(), TEST_SCHEDULE ); Assert.assertEquals(retrievedFlowConfig.getTemplateUris(), TEST_TEMPLATE_URI); // Add this asssert when getFlowSpec() is changed to return the raw flow spec //Assert.assertEquals(flowConfig.getProperties().size(), 2); Assert.assertEquals(retrievedFlowConfig.getProperties().get("param1"), "value1b"); Assert.assertEquals(retrievedFlowConfig.getProperties().get("param2"), "value2b"); }
@Test (dependsOnMethods = "testCreateAgain") public void testGet() throws Exception { FlowId flowId = new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME); FlowConfig flowConfig = _client.getFlowConfig(flowId); Assert.assertEquals(flowConfig.getId().getFlowGroup(), TEST_GROUP_NAME); Assert.assertEquals(flowConfig.getId().getFlowName(), TEST_FLOW_NAME); Assert.assertEquals(flowConfig.getSchedule().getCronSchedule(), TEST_SCHEDULE ); Assert.assertEquals(flowConfig.getTemplateUris(), TEST_TEMPLATE_URI); Assert.assertFalse(flowConfig.getSchedule().isRunImmediately()); // Add this asssert back when getFlowSpec() is changed to return the raw flow spec //Assert.assertEquals(flowConfig.getProperties().size(), 1); Assert.assertEquals(flowConfig.getProperties().get("param1"), "value1"); }
/** * Create a flow configuration * It differs from {@link FlowConfigClient} in a way that it returns FlowStatusId, * which can be used to find the FlowExecutionId * @param flowConfig FlowConfig to be used to create the flow * @return FlowStatusId * @throws RemoteInvocationException */ public FlowStatusId createFlowConfig(FlowConfig flowConfig) throws RemoteInvocationException { LOG.debug("createFlowConfig with groupName " + flowConfig.getId().getFlowGroup() + " flowName " + flowConfig.getId().getFlowName()); CreateIdRequest<ComplexResourceKey<FlowId, FlowStatusId>, FlowConfig> request = _flowconfigsV2RequestBuilders.create().input(flowConfig).build(); ResponseFuture<IdResponse<ComplexResourceKey<FlowId, FlowStatusId>>> flowConfigResponseFuture = _restClient.get().sendRequest(request); return createFlowStatusId(flowConfigResponseFuture.getResponse().getLocation().toString()); }
@Test public void testCreate() throws Exception { Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1"); FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE). setRunImmediately(true)) .setProperties(new StringMap(flowProperties)); _client.createFlowConfig(flowConfig); }
@Test (dependsOnMethods = "testGet") public void testUpdate() throws Exception { logger.info("+++++++++++++++++++ testUpdate START"); // Update on one node and retrieve from another FlowId flowId = new FlowId().setFlowGroup(TEST_GROUP_NAME_1).setFlowName(TEST_FLOW_NAME_1); Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1b"); flowProperties.put("param2", "value2b"); flowProperties.put(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY, TEST_SOURCE_NAME); flowProperties.put(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY, TEST_SINK_NAME); FlowConfig flowConfig = new FlowConfig() .setId(new FlowId().setFlowGroup(TEST_GROUP_NAME_1).setFlowName(TEST_FLOW_NAME_1)) .setTemplateUris(TEST_TEMPLATE_URI_1).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE_1)) .setProperties(new StringMap(flowProperties)); this.node1FlowConfigClient.updateFlowConfig(flowConfig); FlowConfig retrievedFlowConfig = this.node2FlowConfigClient.getFlowConfig(flowId); Assert.assertEquals(retrievedFlowConfig.getId().getFlowGroup(), TEST_GROUP_NAME_1); Assert.assertEquals(retrievedFlowConfig.getId().getFlowName(), TEST_FLOW_NAME_1); Assert.assertEquals(retrievedFlowConfig.getSchedule().getCronSchedule(), TEST_SCHEDULE_1); Assert.assertEquals(retrievedFlowConfig.getTemplateUris(), TEST_TEMPLATE_URI_1); Assert.assertFalse(retrievedFlowConfig.getSchedule().isRunImmediately()); Assert.assertEquals(retrievedFlowConfig.getProperties().get("param1"), "value1b"); Assert.assertEquals(retrievedFlowConfig.getProperties().get("param2"), "value2b"); logger.info("+++++++++++++++++++ testUpdate END"); }
@Test (dependsOnMethods = "testCreateAgain") public void testGet() throws Exception { FlowId flowId = new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME); FlowConfig flowConfig = this.flowConfigClient.getFlowConfig(flowId); Assert.assertEquals(flowConfig.getId().getFlowGroup(), TEST_GROUP_NAME); Assert.assertEquals(flowConfig.getId().getFlowName(), TEST_FLOW_NAME); Assert.assertEquals(flowConfig.getSchedule().getCronSchedule(), TEST_SCHEDULE); Assert.assertEquals(flowConfig.getTemplateUris(), TEST_TEMPLATE_URI); Assert.assertFalse(flowConfig.getSchedule().isRunImmediately()); // Add this assert back when getFlowSpec() is changed to return the raw flow spec //Assert.assertEquals(flowConfig.getProperties().size(), 1); Assert.assertEquals(flowConfig.getProperties().get("param1"), "value1"); }
/** * Method to handle add flow config message forwarded by Helix (Standby) node. * In load balance mode, the FlowCatalog I/O was handled on standby when receiving Restli, so only need to handle * {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onUpdateSpec(Spec)} part. * Otherwise, we have to handle both FlowCatalog I/O and {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onUpdateSpec(Spec)}. * * Please refer to {@link FlowConfigResourceLocalHandler#updateFlowConfig(FlowId, FlowConfig)}. It will handle both FlowCatalog I/O and * {@link org.apache.gobblin.runtime.api.SpecCatalogListener#onUpdateSpec(Spec)} in non-balance mode. */ private void handleUpdate(String msg) throws IOException { FlowConfig config = FlowConfigUtils.deserializeFlowConfig(msg); if (flowCatalogLocalCommit) { // in balance mode, flow spec is already updated in flow catalog on standby node. FlowSpec flowSpec = FlowConfigResourceLocalHandler.createFlowSpecForConfig(config); log.info("Only handle update {} scheduling because flow catalog is committed locally on standby.", flowSpec); jobScheduler.onUpdateSpec(flowSpec); } else { resourceHandler.updateFlowConfig(config.getId(), config); } }
@Test (dependsOnMethods = "testCreate") public void testCreateAgain() throws Exception { Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1"); FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE)) .setProperties(new StringMap(flowProperties)); try { _client.createFlowConfig(flowConfig); } catch (RestLiResponseException e) { Assert.fail("Create Again should pass without complaining that the spec already exists."); } }
.addPrimitive(ConfigurationKeys.FLOW_GROUP_KEY, flowConfig.getId().getFlowGroup()) .addPrimitive(ConfigurationKeys.FLOW_NAME_KEY, flowConfig.getId().getFlowName()); if (flowConfig.hasSchedule()) { Schedule schedule = flowConfig.getSchedule(); configBuilder.addPrimitive(ConfigurationKeys.JOB_SCHEDULE_KEY, schedule.getCronSchedule()); configBuilder.addPrimitive(ConfigurationKeys.FLOW_RUN_IMMEDIATELY, schedule.isRunImmediately()); configWithFallback = config.withFallback(ConfigFactory.parseString(flowConfig.getProperties().toString()).resolve()); } catch (Exception e) { configWithFallback = config.withFallback(ConfigFactory.parseMap(flowConfig.getProperties())); URI templateURI = new URI(flowConfig.getTemplateUris()); return FlowSpec.builder().withConfig(configWithFallback).withTemplate(templateURI).build(); } catch (URISyntaxException e) { throw new FlowConfigLoggedException(HttpStatus.S_400_BAD_REQUEST, "bad URI " + flowConfig.getTemplateUris(), e);
/** * Update a flow configuration * @param flowConfig flow configuration attributes * @throws RemoteInvocationException */ public void updateFlowConfig(FlowConfig flowConfig) throws RemoteInvocationException { LOG.debug("updateFlowConfig with groupName " + flowConfig.getId().getFlowGroup() + " flowName " + flowConfig.getId().getFlowName()); FlowId flowId = new FlowId().setFlowGroup(flowConfig.getId().getFlowGroup()) .setFlowName(flowConfig.getId().getFlowName()); UpdateRequest<FlowConfig> updateRequest = _flowconfigsRequestBuilders.update().id(new ComplexResourceKey<>(flowId, new EmptyRecord())) .input(flowConfig).build(); ResponseFuture<EmptyRecord> response = _restClient.get().sendRequest(updateRequest); response.getResponse(); }