private FlowStatusId createFlowStatusId(String locationHeader) { Matcher matcher = flowStatusIdParams.matcher(locationHeader); matcher.find(); String allFields = matcher.group("flowStatusIdParams"); String[] flowStatusIdParams = allFields.split(","); Map<String, String> paramsMap = new HashMap<>(); for (String flowStatusIdParam : flowStatusIdParams) { paramsMap.put(flowStatusIdParam.split(":")[0], flowStatusIdParam.split(":")[1]); } FlowStatusId flowStatusId = new FlowStatusId() .setFlowName(paramsMap.get("flowName")) .setFlowGroup(paramsMap.get("flowGroup")); if (paramsMap.containsKey("flowExecutionId")) { flowStatusId.setFlowExecutionId(Long.parseLong(paramsMap.get("flowExecutionId"))); } return flowStatusId; }
/** * Retrieve the FlowStatus with the given key * @param key flow status id key containing group name and flow name * @return {@link FlowStatus} with flow status for the latest execution of the flow */ @Override public FlowStatus get(ComplexResourceKey<FlowStatusId, EmptyRecord> key) { String flowGroup = key.getKey().getFlowGroup(); String flowName = key.getKey().getFlowName(); long flowExecutionId = key.getKey().getFlowExecutionId(); LOG.info("Get called with flowGroup " + flowGroup + " flowName " + flowName + " flowExecutionId " + flowExecutionId); org.apache.gobblin.service.monitoring.FlowStatus flowStatus = _flowStatusGenerator.getFlowStatus(flowName, flowGroup, flowExecutionId); // this returns null to raise a 404 error if flowStatus is null return convertFlowStatus(flowStatus); }
/** * Get a flow status * @param flowStatusId identifier of flow status to get * @return a {@link FlowStatus} with the flow status * @throws RemoteInvocationException */ public FlowStatus getFlowStatus(FlowStatusId flowStatusId) throws RemoteInvocationException { LOG.debug("getFlowConfig with groupName " + flowStatusId.getFlowGroup() + " flowName " + flowStatusId.getFlowName()); GetRequest<FlowStatus> getRequest = _flowstatusesRequestBuilders.get() .id(new ComplexResourceKey<>(flowStatusId, new EmptyRecord())).build(); Response<FlowStatus> response = _restClient.get().sendRequest(getRequest).getResponse(); return response.getEntity(); }
_listOfJobStatusLists.add(jobStatusList); FlowStatusId flowId = new FlowStatusId().setFlowGroup("fgroup1").setFlowName("flow1").setFlowExecutionId(0); FlowStatus flowStatus = _client.getFlowStatus(flowId); Assert.assertEquals(flowStatus.getId().getFlowGroup(), "fgroup1"); Assert.assertEquals(flowStatus.getId().getFlowName(), "flow1"); Assert.assertEquals(flowStatus.getExecutionStatistics().getExecutionStartTime().longValue(), 1000L); Assert.assertEquals(flowStatus.getExecutionStatistics().getExecutionEndTime().longValue(), 6000L);
/** * Delete a flow configuration * @param flowId identifier of flow configuration to delete * @throws RemoteInvocationException */ public void deleteFlowConfig(FlowId flowId) throws RemoteInvocationException { LOG.debug("deleteFlowConfig with groupName {}, flowName {}", flowId.getFlowGroup(), flowId.getFlowName()); DeleteRequest<FlowConfig> deleteRequest = _flowconfigsV2RequestBuilders.delete() .id(new ComplexResourceKey<>(flowId, new FlowStatusId())).build(); ResponseFuture<EmptyRecord> response = _restClient.get().sendRequest(deleteRequest); response.getResponse(); }
_listOfJobStatusLists.add(jobStatusList); FlowStatusId flowId = new FlowStatusId().setFlowGroup("fgroup1").setFlowName("flow1").setFlowExecutionId(0); FlowStatus flowStatus = _client.getFlowStatus(flowId); Assert.assertEquals(flowStatus.getId().getFlowGroup(), "fgroup1"); Assert.assertEquals(flowStatus.getId().getFlowName(), "flow1"); Assert.assertEquals(flowStatus.getExecutionStatistics().getExecutionStartTime().longValue(), 1000L); Assert.assertEquals(flowStatus.getExecutionStatistics().getExecutionEndTime().longValue(), 6000L);
FlowStatus flowStatus = _client.getLatestFlowStatus(flowId); Assert.assertEquals(flowStatus.getId().getFlowGroup(), "fgroup1"); Assert.assertEquals(flowStatus.getId().getFlowName(), "flow1"); Assert.assertEquals(flowStatus.getExecutionStatistics().getExecutionStartTime().longValue(), 2000L); Assert.assertEquals(flowStatus.getExecutionStatistics().getExecutionEndTime().longValue(), 6000L);
/** * Delete a flow configuration * @param flowId identifier of flow configuration to delete * @throws RemoteInvocationException */ public void deleteFlowConfigWithStateStore(FlowId flowId) throws RemoteInvocationException { LOG.debug("deleteFlowConfig and state store with groupName " + flowId.getFlowGroup() + " flowName " + flowId.getFlowName()); DeleteRequest<FlowConfig> deleteRequest = _flowconfigsV2RequestBuilders.delete() .id(new ComplexResourceKey<>(flowId, new FlowStatusId())).setHeader(DELETE_STATE_STORE_KEY, Boolean.TRUE.toString()).build(); ResponseFuture<EmptyRecord> response = _restClient.get().sendRequest(deleteRequest); response.getResponse(); }
_listOfJobStatusLists.add(jobStatusList); FlowStatusId flowId = new FlowStatusId().setFlowGroup("fgroup1").setFlowName("flow1").setFlowExecutionId(0); FlowStatus flowStatus = _client.getFlowStatus(flowId); Assert.assertEquals(flowStatus.getId().getFlowGroup(), "fgroup1"); Assert.assertEquals(flowStatus.getId().getFlowName(), "flow1"); Assert.assertEquals(flowStatus.getExecutionStatistics().getExecutionStartTime().longValue(), 1000L); Assert.assertEquals(flowStatus.getExecutionStatistics().getExecutionEndTime().longValue(), 6000L);
@Override /** * Add flowConfig locally and trigger all listeners iff @param triggerListener is set to true */ public CreateResponse createFlowConfig(FlowConfig flowConfig, boolean triggerListener) throws FlowConfigLoggedException { log.info("[GAAS-REST] Create called with flowGroup " + flowConfig.getId().getFlowGroup() + " flowName " + flowConfig.getId().getFlowName()); FlowSpec flowSpec = createFlowSpecForConfig(flowConfig); this.flowCatalog.put(flowSpec, triggerListener); FlowStatusId flowStatusId = new FlowStatusId() .setFlowName(flowSpec.getConfigAsProperties().getProperty(ConfigurationKeys.FLOW_NAME_KEY)) .setFlowGroup(flowSpec.getConfigAsProperties().getProperty(ConfigurationKeys.FLOW_GROUP_KEY)); if (flowSpec.getConfigAsProperties().containsKey(ConfigurationKeys.FLOW_EXECUTION_ID_KEY)) { flowStatusId.setFlowExecutionId(Long.valueOf(flowSpec.getConfigAsProperties().getProperty(ConfigurationKeys.FLOW_EXECUTION_ID_KEY))); } else { flowStatusId.setFlowExecutionId(-1L); } return new CreateResponse(new ComplexResourceKey<>(flowConfig.getId(), flowStatusId), HttpStatus.S_201_CREATED); } }
@Test public void testCheckFlowExecutionId() throws Exception { Map<String, String> flowProperties = Maps.newHashMap(); flowProperties.put("param1", "value1"); FlowConfig flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setProperties(new StringMap(flowProperties)); FlowStatusId flowStatusId =_client.createFlowConfig(flowConfig); Assert.assertEquals(TEST_GROUP_NAME, flowStatusId.getFlowGroup()); Assert.assertEquals(TEST_FLOW_NAME, flowStatusId.getFlowName()); Assert.assertTrue(flowStatusId.getFlowExecutionId() != -1); flowConfig = new FlowConfig().setId(new FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME)) .setTemplateUris(TEST_TEMPLATE_URI).setProperties(new StringMap(flowProperties)) .setSchedule(new Schedule().setCronSchedule(TEST_SCHEDULE).setRunImmediately(true)); Assert.assertEquals(_client.createFlowConfig(flowConfig).getFlowExecutionId().longValue(), -1L); }
/** * Get a flow status * @param flowStatusId identifier of flow status to get * @return a {@link FlowStatus} with the flow status * @throws RemoteInvocationException */ public FlowStatus getFlowStatus(FlowStatusId flowStatusId) throws RemoteInvocationException { LOG.debug("getFlowConfig with groupName " + flowStatusId.getFlowGroup() + " flowName " + flowStatusId.getFlowName()); GetRequest<FlowStatus> getRequest = _flowstatusesRequestBuilders.get() .id(new ComplexResourceKey<>(flowStatusId, new EmptyRecord())).build(); Response<FlowStatus> response = _restClient.get().sendRequest(getRequest).getResponse(); return response.getEntity(); }
/** * Get a flow configuration * @param flowId identifier of flow configuration to get * @return a {@link FlowConfig} with the flow configuration * @throws RemoteInvocationException */ public FlowConfig getFlowConfig(FlowId flowId) throws RemoteInvocationException { LOG.debug("getFlowConfig with groupName " + flowId.getFlowGroup() + " flowName " + flowId.getFlowName()); GetRequest<FlowConfig> getRequest = _flowconfigsV2RequestBuilders.get() .id(new ComplexResourceKey<>(flowId, new FlowStatusId())).build(); Response<FlowConfig> response = _restClient.get().sendRequest(getRequest).getResponse(); return response.getEntity(); }
.setId(new FlowStatusId().setFlowGroup(flowId.getFlowGroup()).setFlowName(flowId.getFlowName()) .setFlowExecutionId(monitoringFlowStatus.getFlowExecutionId())) .setExecutionStatistics(new FlowStatistics().setExecutionStartTime(flowStartTime) .setExecutionEndTime(flowEndTime))
/** * Update a flow configuration * @param flowConfig flow configuration attributes * @throws RemoteInvocationException */ public void updateFlowConfig(FlowConfig flowConfig) throws RemoteInvocationException { LOG.debug("updateFlowConfig with groupName " + flowConfig.getId().getFlowGroup() + " flowName " + flowConfig.getId().getFlowName()); FlowId flowId = new FlowId().setFlowGroup(flowConfig.getId().getFlowGroup()) .setFlowName(flowConfig.getId().getFlowName()); UpdateRequest<FlowConfig> updateRequest = _flowconfigsV2RequestBuilders.update().id(new ComplexResourceKey<>(flowId, new FlowStatusId())) .input(flowConfig).build(); ResponseFuture<EmptyRecord> response = _restClient.get().sendRequest(updateRequest); response.getResponse(); }
private FlowStatusId createFlowStatusId(String locationHeader) { Matcher matcher = flowStatusIdParams.matcher(locationHeader); matcher.find(); String allFields = matcher.group("flowStatusIdParams"); String[] flowStatusIdParams = allFields.split(","); Map<String, String> paramsMap = new HashMap<>(); for (String flowStatusIdParam : flowStatusIdParams) { paramsMap.put(flowStatusIdParam.split(":")[0], flowStatusIdParam.split(":")[1]); } FlowStatusId flowStatusId = new FlowStatusId() .setFlowName(paramsMap.get("flowName")) .setFlowGroup(paramsMap.get("flowGroup")); if (paramsMap.containsKey("flowExecutionId")) { flowStatusId.setFlowExecutionId(Long.parseLong(paramsMap.get("flowExecutionId"))); } return flowStatusId; }
/** * Delete a flow configuration * @param flowId identifier of flow configuration to delete * @throws RemoteInvocationException */ public void deleteFlowConfig(FlowId flowId) throws RemoteInvocationException { LOG.debug("deleteFlowConfig with groupName {}, flowName {}", flowId.getFlowGroup(), flowId.getFlowName()); DeleteRequest<FlowConfig> deleteRequest = _flowconfigsV2RequestBuilders.delete() .id(new ComplexResourceKey<>(flowId, new FlowStatusId())).build(); ResponseFuture<EmptyRecord> response = _restClient.get().sendRequest(deleteRequest); response.getResponse(); }
/** * Delete a flow configuration * @param flowId identifier of flow configuration to delete * @throws RemoteInvocationException */ public void deleteFlowConfigWithStateStore(FlowId flowId) throws RemoteInvocationException { LOG.debug("deleteFlowConfig and state store with groupName " + flowId.getFlowGroup() + " flowName " + flowId.getFlowName()); DeleteRequest<FlowConfig> deleteRequest = _flowconfigsV2RequestBuilders.delete() .id(new ComplexResourceKey<>(flowId, new FlowStatusId())).setHeader(DELETE_STATE_STORE_KEY, Boolean.TRUE.toString()).build(); ResponseFuture<EmptyRecord> response = _restClient.get().sendRequest(deleteRequest); response.getResponse(); }
/** * Get a flow configuration * @param flowId identifier of flow configuration to get * @return a {@link FlowConfig} with the flow configuration * @throws RemoteInvocationException */ public FlowConfig getFlowConfig(FlowId flowId) throws RemoteInvocationException { LOG.debug("getFlowConfig with groupName " + flowId.getFlowGroup() + " flowName " + flowId.getFlowName()); GetRequest<FlowConfig> getRequest = _flowconfigsV2RequestBuilders.get() .id(new ComplexResourceKey<>(flowId, new FlowStatusId())).build(); Response<FlowConfig> response = _restClient.get().sendRequest(getRequest).getResponse(); return response.getEntity(); }
/** * Update a flow configuration * @param flowConfig flow configuration attributes * @throws RemoteInvocationException */ public void updateFlowConfig(FlowConfig flowConfig) throws RemoteInvocationException { LOG.debug("updateFlowConfig with groupName " + flowConfig.getId().getFlowGroup() + " flowName " + flowConfig.getId().getFlowName()); FlowId flowId = new FlowId().setFlowGroup(flowConfig.getId().getFlowGroup()) .setFlowName(flowConfig.getId().getFlowName()); UpdateRequest<FlowConfig> updateRequest = _flowconfigsV2RequestBuilders.update().id(new ComplexResourceKey<>(flowId, new FlowStatusId())) .input(flowConfig).build(); ResponseFuture<EmptyRecord> response = _restClient.get().sendRequest(updateRequest); response.getResponse(); }