public JsonProtoModule() { super("ConductorJsonProtoModule"); addSerializer(Any.class, new AnySerializer()); addDeserializer(Any.class, new AnyDeserializer()); } }
/** * Get external uri for the payload * * @param operation the type of {@link Operation} to be performed * @param payloadType the {@link PayloadType} at the external uri * @param path the path for which the external storage location is to be populated * @return the external uri at which the payload is stored/to be stored */ public ExternalStorageLocation getExternalStorageLocation(Operation operation, PayloadType payloadType, String path) { return externalPayloadStorage.getLocation(operation, payloadType, path); } }
@Override public ObjectMapper get() { final ObjectMapper objectMapper = new ObjectMapper(); objectMapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); objectMapper.configure(DeserializationFeature.FAIL_ON_IGNORED_PROPERTIES, false); objectMapper.configure(DeserializationFeature.FAIL_ON_NULL_FOR_PRIMITIVES, false); objectMapper.setSerializationInclusion(JsonInclude.Include.NON_NULL); objectMapper.setSerializationInclusion(JsonInclude.Include.NON_EMPTY); objectMapper.registerModule(new JsonProtoModule()); return objectMapper; } }
@Before public void setup() { objectMapper = new JsonMapperProvider().get(); }
key, paramPath); errorList.add( message ); } else if (EnvUtils.isEnvironmentVariable(paramPath)) { for (SystemParameters systemParameters : SystemParameters.values()) { if (systemParameters.name().equals(paramPath)) { isPredefinedEnum = true; break; String sysValue = EnvUtils.getSystemParametersValue(paramPath,"" ); if (sysValue == null) { String errorMessage = String.format("environment variable: %s for given task: %s" +
/** * This method is not intended to be used in the client. * The client makes a request to the server to get the {@link ExternalStorageLocation} */ @Override public ExternalStorageLocation getLocation(Operation operation, PayloadType payloadType, String path) { String uri; switch (payloadType) { case WORKFLOW_INPUT: case WORKFLOW_OUTPUT: uri = "workflow"; break; case TASK_INPUT: case TASK_OUTPUT: uri = "tasks"; break; default: throw new ConductorClientException(String.format("Invalid payload type: %s for operation: %s", payloadType.toString(), operation.toString())); } return clientBase.getForEntity(String.format("%s/externalstoragelocation", uri), new Object[]{"path", path}, ExternalStorageLocation.class); }
@Test public void testExtractParamPathComponentsWithMissingEnvVariable() { WorkflowDef workflowDef = constructWorkflowDef(); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map<String, Object> inputParam = new HashMap<>(); inputParam.put("taskId", "${CPEWF_TASK_ID} ${NETFLIX_STACK}"); workflowTask_1.setInputParameters(inputParam); List<WorkflowTask> tasks = new ArrayList<>(); tasks.add(workflowTask_1); workflowDef.setTasks(tasks); List<String> results = ConstraintParamUtil.validateInputParam(inputParam, "task_1", workflowDef); assertEquals(results.size(), 0); }
private static List<String> extractListInputParam(String key, List<?> values, String taskName, WorkflowDef workflow) { ArrayList<String> errorList = new ArrayList<>(); for (Object listVal : values) { if (listVal instanceof String) { errorList.addAll(extractParamPathComponentsFromString(key, listVal.toString(), taskName, workflow)); } else if (listVal instanceof Map) { errorList.addAll( validateInputParam((Map<String, Object>) listVal, taskName, workflow)); } else if (listVal instanceof List) { errorList.addAll(extractListInputParam(key, (List<?>) listVal, taskName, workflow)); } } return errorList; }
@VisibleForTesting String uploadHelper(byte[] payloadBytes, long payloadSize, ExternalPayloadStorage.PayloadType payloadType) { ExternalStorageLocation location = externalPayloadStorage.getLocation(ExternalPayloadStorage.Operation.WRITE, payloadType, ""); externalPayloadStorage.upload(location.getPath(), new ByteArrayInputStream(payloadBytes), payloadSize); return location.getPath(); }
/** * Populates the workflow output from external payload storage if the external storage path is specified. * * @param workflow the workflow for which the output is to be populated. */ private void populateWorkflowOutput(Workflow workflow) { if (StringUtils.isNotBlank(workflow.getExternalOutputPayloadStoragePath())) { WorkflowTaskMetrics.incrementExternalPayloadUsedCount(workflow.getWorkflowName(), ExternalPayloadStorage.Operation.READ.name(), ExternalPayloadStorage.PayloadType.WORKFLOW_OUTPUT.name()); workflow.setOutput(downloadFromExternalStorage(ExternalPayloadStorage.PayloadType.WORKFLOW_OUTPUT, workflow.getExternalOutputPayloadStoragePath())); } }
public static boolean isEnvironmentVariable(String test) { for (SystemParameters c : SystemParameters.values()) { if (c.name().equals(test)) { return true; } } String value = Optional.ofNullable(System.getProperty(test)) .orElseGet(() -> Optional.ofNullable(System.getenv(test)) .orElse(null)); return value != null; }
/** * Download the payload from the given path * * @param path the relative path of the payload in the {@link ExternalPayloadStorage} * @return the payload object * @throws ApplicationException in case of JSON parsing errors or download errors */ @SuppressWarnings("unchecked") public Map<String, Object> downloadPayload(String path) { try (InputStream inputStream = externalPayloadStorage.download(path)) { return objectMapper.readValue(IOUtils.toString(inputStream), Map.class); } catch (IOException e) { logger.error("Unable to download payload from external storage path: {}", path, e); throw new ApplicationException(ApplicationException.Code.INTERNAL_ERROR, e); } }
/** * Uses the {@link PayloadStorage} for storing large payloads. * Gets the uri for storing the payload from the server and then uploads to this location. * * @param payloadType the {@link com.netflix.conductor.common.utils.ExternalPayloadStorage.PayloadType} to be uploaded * @param payloadBytes the byte array containing the payload * @param payloadSize the size of the payload * @return the path where the payload is stored in external storage */ protected String uploadToExternalPayloadStorage(ExternalPayloadStorage.PayloadType payloadType, byte[] payloadBytes, long payloadSize) { Preconditions.checkArgument(payloadType.equals(ExternalPayloadStorage.PayloadType.WORKFLOW_INPUT) || payloadType.equals(ExternalPayloadStorage.PayloadType.TASK_OUTPUT), "Payload type must be workflow input or task output"); ExternalStorageLocation externalStorageLocation = payloadStorage.getLocation(ExternalPayloadStorage.Operation.WRITE, payloadType, ""); payloadStorage.upload(externalStorageLocation.getUri(), new ByteArrayInputStream(payloadBytes), payloadSize); return externalStorageLocation.getPath(); }
protected ClientBase(ClientConfig config, ConductorClientConfiguration clientConfiguration, ClientHandler handler) { objectMapper = new JsonMapperProvider().get(); JacksonJsonProvider provider = new JacksonJsonProvider(objectMapper); config.getSingletons().add(provider); if (handler == null) { this.client = Client.create(config); } else { this.client = new Client(handler, config); } conductorClientConfiguration = clientConfiguration; payloadStorage = new PayloadStorage(this); }
@Test public void testExtractParamPathComponentsWithInputParamEmpty() { WorkflowDef workflowDef = constructWorkflowDef(); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map<String, Object> inputParam = new HashMap<>(); inputParam.put("taskId", ""); workflowTask_1.setInputParameters(inputParam); List<WorkflowTask> tasks = new ArrayList<>(); tasks.add(workflowTask_1); workflowDef.setTasks(tasks); List<String> results = ConstraintParamUtil.validateInputParam(inputParam,"task_1", workflowDef); assertEquals(results.size(), 1); }
/** * Populates the task input from external payload storage if the external storage path is specified. * * @param task the task for which the input is to be populated. */ private void populateTaskInput(Task task) { if (StringUtils.isNotBlank(task.getExternalInputPayloadStoragePath())) { WorkflowTaskMetrics.incrementExternalPayloadUsedCount(task.getTaskDefName(), ExternalPayloadStorage.Operation.READ.name(), ExternalPayloadStorage.PayloadType.TASK_INPUT.name()); task.setInputData(downloadFromExternalStorage(ExternalPayloadStorage.PayloadType.TASK_INPUT, task.getExternalInputPayloadStoragePath())); task.setExternalInputPayloadStoragePath(null); } }
@Before public void setUp() { executionDAO = mock(ExecutionDAO.class); indexDAO = mock(IndexDAO.class); objectMapper = new JsonMapperProvider().get(); executionDAOFacade = new ExecutionDAOFacade(executionDAO, indexDAO, objectMapper); }
@Test public void testExtractParamPathComponentsWithInputFieldWithSpace() { WorkflowDef workflowDef = constructWorkflowDef(); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map<String, Object> inputParam = new HashMap<>(); inputParam.put("taskId", "${CPEWF_TASK_ID} ${workflow.input.status sta}"); workflowTask_1.setInputParameters(inputParam); List<WorkflowTask> tasks = new ArrayList<>(); tasks.add(workflowTask_1); workflowDef.setTasks(tasks); List<String> results = ConstraintParamUtil.validateInputParam(inputParam,"task_1", workflowDef); assertEquals(results.size(), 1); }
@Test public void testExtractParamPathComponentsWithValidEnvVariable() { WorkflowDef workflowDef = constructWorkflowDef(); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map<String, Object> inputParam = new HashMap<>(); inputParam.put("taskId", "${CPEWF_TASK_ID} ${workflow.input.status}"); workflowTask_1.setInputParameters(inputParam); List<WorkflowTask> tasks = new ArrayList<>(); tasks.add(workflowTask_1); workflowDef.setTasks(tasks); List<String> results = ConstraintParamUtil.validateInputParam(inputParam,"task_1", workflowDef); assertEquals(results.size(), 0); }
@Test public void testExtractParamPathComponents() { WorkflowDef workflowDef = constructWorkflowDef(); WorkflowTask workflowTask_1 = new WorkflowTask(); workflowTask_1.setName("task_1"); workflowTask_1.setTaskReferenceName("task_1"); workflowTask_1.setType(TaskType.TASK_TYPE_SIMPLE); Map<String, Object> inputParam = new HashMap<>(); inputParam.put("taskId", "${CPEWF_TASK_ID}"); workflowTask_1.setInputParameters(inputParam); List<WorkflowTask> tasks = new ArrayList<>(); tasks.add(workflowTask_1); workflowDef.setTasks(tasks); List<String> results = ConstraintParamUtil.validateInputParam(inputParam, "task_1", workflowDef); assertEquals(results.size(), 0); }