context.getNamespace()); PluginContext pluginContext = new PipelinePluginContext(context, workflowMetrics, spec.isStageLoggingEnabled(), spec.isProcessTimingEnabled()); for (ActionSpec actionSpec : spec.getEndingActions()) { String stageName = actionSpec.getName(); postActions.put(stageName, pluginContext.newPluginInstance(stageName, macroEvaluator)); stageSpecs.put(stageName, StageSpec.builder(stageName, actionSpec.getPluginSpec()) .setStageLoggingEnabled(spec.isStageLoggingEnabled()) .setProcessTimingEnabled(spec.isProcessTimingEnabled()) .build()); for (StageSpec stageSpec : spec.getStages()) { String stageName = stageSpec.getName(); stageSpecs.put(stageName, stageSpec);
return new BatchPhaseSpec(programName, phase, spec.getResources(), spec.getDriverResources(), spec.getClientResources(), spec.isStageLoggingEnabled(), spec.isProcessTimingEnabled(), phaseConnectorDatasets, spec.getNumOfRecordsPreview(), spec.getProperties(), !plan.getConditionPhaseBranches().isEmpty());
@TransactionPolicy(TransactionControl.EXPLICIT) @Override public void initialize(WorkflowContext context) throws Exception { super.initialize(context); postActions = new LinkedHashMap<>(); BatchPipelineSpec batchPipelineSpec = GSON.fromJson(context.getWorkflowSpecification().getProperty("pipeline.spec"), BatchPipelineSpec.class); MacroEvaluator macroEvaluator = new DefaultMacroEvaluator(new BasicArguments(context.getToken(), context.getRuntimeArguments()), context.getLogicalStartTime(), context, context.getNamespace()); postActionSpecs = new HashMap<>(); for (ActionSpec actionSpec : batchPipelineSpec.getEndingActions()) { String name = actionSpec.getName(); postActions.put(name, (PostAction) context.newPluginInstance(name, macroEvaluator)); postActionSpecs.put(name, StageSpec.builder(name, actionSpec.getPluginSpec()) .setProcessTimingEnabled(batchPipelineSpec.isProcessTimingEnabled()) .setStageLoggingEnabled(batchPipelineSpec.isStageLoggingEnabled()) .build()); } }
for (StageSpec stageSpec : spec.getStages()) { stageSpecs.put(stageSpec.getName(), stageSpec); String pluginType = stageSpec.getPlugin().getType();
LineageOperationsProcessor processor = new LineageOperationsProcessor(spec.getConnections(), allStageOperations, noMergeRequiredStages);
for (StageSpec stageSpec : spec.getStages()) { if (BatchSource.PLUGIN_TYPE.equals(stageSpec.getPlugin().getType())) { sourceCount++;