/** * Creates a new instance based on the given {@link WorkflowProgramInfo}. */ public static NameMappedDatasetFramework createFromWorkflowProgramInfo(DatasetFramework datasetFramework, WorkflowProgramInfo info, ApplicationSpecification appSpec) { Set<String> localDatasets = appSpec.getWorkflows().get(info.getName()).getLocalDatasetSpecs().keySet(); return new NameMappedDatasetFramework(datasetFramework, localDatasets, info.getRunId().getId()); }
@Override public WorkflowToken getWorkflowToken() { return workflowProgramInfo.getWorkflowToken(); }
private static Map<String, String> createMetricsTags(@Nullable WorkflowProgramInfo workflowProgramInfo) { if (workflowProgramInfo != null) { return workflowProgramInfo.updateMetricsTags(new HashMap<String, String>()); } return Collections.emptyMap(); } }
/** * Updates the metrics tags based on the information in this class. */ public Map<String, String> updateMetricsTags(Map<String, String> tags) { tags.put(Constants.Metrics.Tag.WORKFLOW, getName()); tags.put(Constants.Metrics.Tag.WORKFLOW_RUN_ID, getRunId().getId()); tags.put(Constants.Metrics.Tag.NODE, getNodeId()); return tags; }
private boolean emitFieldLineage() { if (runtimeContext.getFieldLineageOperations().isEmpty()) { // no operations to emit return false; } ProgramStatus status = context.getState().getStatus(); if (ProgramStatus.COMPLETED != status) { // Spark program failed, so field operations wont be emitted return false; } WorkflowProgramInfo workflowInfo = runtimeContext.getWorkflowInfo(); return workflowInfo == null || !workflowInfo.fieldLineageConsolidationEnabled(); }
WorkflowProgramInfo workflowInfo = WorkflowProgramInfo.create(arguments); DatasetFramework programDatasetFramework = workflowInfo == null ? datasetFramework :
/** * Optionally creates a {@link WorkflowProgramInfo} from the given arguments. If the arguments don't contain * workflow information, {@code null} will be returned. */ @Nullable public static WorkflowProgramInfo create(Arguments arguments) { String workflowName = arguments.getOption(ProgramOptionConstants.WORKFLOW_NAME); String workflowNodeId = arguments.getOption(ProgramOptionConstants.WORKFLOW_NODE_ID); String workflowRunId = arguments.getOption(ProgramOptionConstants.WORKFLOW_RUN_ID); String programNameInWorkflow = arguments.getOption(ProgramOptionConstants.PROGRAM_NAME_IN_WORKFLOW); String workflowToken = arguments.getOption(ProgramOptionConstants.WORKFLOW_TOKEN); boolean consolidateFieldOperations = Boolean.parseBoolean(arguments.getOption(ProgramOptionConstants.ENABLE_FIELD_LINEAGE_CONSOLIDATION)); if (workflowName == null || workflowNodeId == null || workflowRunId == null || workflowToken == null) { return null; } return new WorkflowProgramInfo(workflowName, workflowNodeId, workflowRunId, programNameInWorkflow, GSON.fromJson(workflowToken, BasicWorkflowToken.class), consolidateFieldOperations); }
/** * Updates the metrics tags based on the information in this class. */ public Map<String, String> updateMetricsTags(Map<String, String> tags) { tags.put(Constants.Metrics.Tag.WORKFLOW, getName()); tags.put(Constants.Metrics.Tag.WORKFLOW_RUN_ID, getRunId().getId()); tags.put(Constants.Metrics.Tag.NODE, getNodeId()); return tags; }
private boolean emitFieldLineage() { if (context.getFieldLineageOperations().isEmpty()) { // no operations to emit return false; } ProgramStatus status = context.getState().getStatus(); if (ProgramStatus.COMPLETED != status) { // MapReduce program failed, so field operations wont be emitted return false; } WorkflowProgramInfo workflowInfo = context.getWorkflowInfo(); return workflowInfo == null || !workflowInfo.fieldLineageConsolidationEnabled(); }
WorkflowProgramInfo workflowInfo = WorkflowProgramInfo.create(arguments); DatasetFramework programDatasetFramework = workflowInfo == null ? datasetFramework :
/** * Optionally creates a {@link WorkflowProgramInfo} from the given arguments. If the arguments don't contain * workflow information, {@code null} will be returned. */ @Nullable public static WorkflowProgramInfo create(Arguments arguments) { String workflowName = arguments.getOption(ProgramOptionConstants.WORKFLOW_NAME); String workflowNodeId = arguments.getOption(ProgramOptionConstants.WORKFLOW_NODE_ID); String workflowRunId = arguments.getOption(ProgramOptionConstants.WORKFLOW_RUN_ID); String programNameInWorkflow = arguments.getOption(ProgramOptionConstants.PROGRAM_NAME_IN_WORKFLOW); String workflowToken = arguments.getOption(ProgramOptionConstants.WORKFLOW_TOKEN); boolean consolidateFieldOperations = Boolean.parseBoolean(arguments.getOption(ProgramOptionConstants.ENABLE_FIELD_LINEAGE_CONSOLIDATION)); if (workflowName == null || workflowNodeId == null || workflowRunId == null || workflowToken == null) { return null; } return new WorkflowProgramInfo(workflowName, workflowNodeId, workflowRunId, programNameInWorkflow, GSON.fromJson(workflowToken, BasicWorkflowToken.class), consolidateFieldOperations); }
/** * Creates a new instance based on the given {@link WorkflowProgramInfo}. */ public static NameMappedDatasetFramework createFromWorkflowProgramInfo(DatasetFramework datasetFramework, WorkflowProgramInfo info, ApplicationSpecification appSpec) { Set<String> localDatasets = appSpec.getWorkflows().get(info.getName()).getLocalDatasetSpecs().keySet(); return new NameMappedDatasetFramework(datasetFramework, localDatasets, info.getRunId().getId()); }
@Override public WorkflowToken getWorkflowToken() { return workflowProgramInfo.getWorkflowToken(); }
private static Map<String, String> createMetricsTags(@Nullable WorkflowProgramInfo workflowProgramInfo) { if (workflowProgramInfo != null) { return workflowProgramInfo.updateMetricsTags(new HashMap<String, String>()); } return Collections.emptyMap(); } }
private boolean emitFieldLineage() { if (context.getFieldLineageOperations().isEmpty()) { // no operations to emit return false; } ProgramStatus status = context.getState().getStatus(); if (ProgramStatus.COMPLETED != status) { // MapReduce program failed, so field operations wont be emitted return false; } WorkflowProgramInfo workflowInfo = context.getWorkflowInfo(); return workflowInfo == null || !workflowInfo.fieldLineageConsolidationEnabled(); }
WorkflowProgramInfo workflowInfo = WorkflowProgramInfo.create(arguments); DatasetFramework programDatasetFramework = workflowInfo == null ? datasetFramework :
WorkflowProgramInfo info = new WorkflowProgramInfo(workflowSpec.getName(), node.getNodeId(), workflowRunId.getRun(), node.getNodeId(), (BasicWorkflowToken) token,
private LoggingContext createLoggingContext(ProgramId programId, RunId runId, @Nullable WorkflowProgramInfo workflowProgramInfo) { if (workflowProgramInfo == null) { return new MapReduceLoggingContext(programId.getNamespace(), programId.getApplication(), programId.getProgram(), runId.getId()); } ProgramId workflowProramId = programId.getParent().workflow(workflowProgramInfo.getName()); return new WorkflowProgramLoggingContext(workflowProramId.getNamespace(), workflowProramId.getApplication(), workflowProramId.getProgram(), workflowProgramInfo.getRunId().getId(), ProgramType.MAPREDUCE, programId.getProgram(), runId.getId()); }
/** * Returns the WorkflowToken if the MapReduce program is executed as a part of the Workflow. */ @Override @Nullable public WorkflowToken getWorkflowToken() { return workflowProgramInfo == null ? null : workflowProgramInfo.getWorkflowToken(); }
/** * Creates metrics tags to be used for the Spark execution. */ private static Map<String, String> createMetricsTags(@Nullable WorkflowProgramInfo workflowProgramInfo) { Map<String, String> tags = Maps.newHashMap(); // todo: use proper spark instance id. For now we have to emit smth for test framework's waitFor metric to work tags.put(Constants.Metrics.Tag.INSTANCE_ID, "0"); if (workflowProgramInfo != null) { workflowProgramInfo.updateMetricsTags(tags); } return tags; } }