/** * Instantiates and initializes the plugin for the stage. * * @param stageInfo the stage info * @return the initialized Transformation * @throws InstantiationException if the plugin for the stage could not be instantiated * @throws Exception if there was a problem initializing the plugin */ private <T extends Transformation & StageLifecycle<BatchRuntimeContext>> Transformation getInitializedTransformation(StageSpec stageInfo) throws Exception { BatchRuntimeContext runtimeContext = createRuntimeContext(stageInfo); T plugin = pluginInstantiator.newPluginInstance(stageInfo.getName(), macroEvaluator); plugin.initialize(runtimeContext); return plugin; }
if (BatchSource.PLUGIN_TYPE.equals(pluginType) || isConnectorSource) { BatchConfigurable<BatchSourceContext> batchSource = pluginInstantiator.newPluginInstance(stageName, evaluator); ContextProvider<MapReduceBatchContext> contextProvider = new MapReduceBatchContextProvider(context, pipelineRuntime, stageSpec, connectorDatasets); AlertPublisher.PLUGIN_TYPE.equals(pluginType) || isConnectorSink) { BatchConfigurable<BatchSinkContext> batchSink = pluginInstantiator.newPluginInstance(stageName, evaluator); ContextProvider<MapReduceBatchContext> contextProvider = new MapReduceBatchContextProvider(context, pipelineRuntime, stageSpec, connectorDatasets); Transform<?, ?> transform = pluginInstantiator.newPluginInstance(stageName, evaluator); ContextProvider<MapReduceBatchContext> contextProvider = new MapReduceBatchContextProvider(context, pipelineRuntime, stageSpec, connectorDatasets); final BatchAggregator<?, ?, ?> aggregator = pluginInstantiator.newPluginInstance(stageName, evaluator); ContextProvider<DefaultAggregatorContext> contextProvider = new AggregatorContextProvider(pipelineRuntime, stageSpec, context.getAdmin()); final BatchJoiner<?, ?, ?> batchJoiner = pluginInstantiator.newPluginInstance(stageName, evaluator); ContextProvider<DefaultJoinerContext> contextProvider = new JoinerContextProvider(pipelineRuntime, stageSpec, context.getAdmin());
if (BatchSource.PLUGIN_TYPE.equals(pluginType) || isConnectorSource) { BatchConfigurable<BatchSourceContext> batchSource = pluginInstantiator.newPluginInstance(stageName, evaluator); ContextProvider<BatchSourceContext> contextProvider = new ContextProvider<BatchSourceContext>() { Transform transform = pluginInstantiator.newPluginInstance(stageName, evaluator); ContextProvider<StageSubmitterContext> contextProvider = new ContextProvider<StageSubmitterContext>() { BatchConfigurable<BatchSinkContext> batchSink = pluginInstantiator.newPluginInstance(stageName, evaluator); ContextProvider<BatchSinkContext> contextProvider = new ContextProvider<BatchSinkContext>() { @Override BatchConfigurable<SparkPluginContext> sparkSink = pluginInstantiator.newPluginInstance(stageName, evaluator); ContextProvider<SparkPluginContext> contextProvider = new ContextProvider<SparkPluginContext>() { BatchAggregator aggregator = pluginInstantiator.newPluginInstance(stageName, evaluator); ContextProvider<DefaultAggregatorContext> contextProvider = new AggregatorContextProvider(pipelineRuntime, stageSpec, admin); BatchJoiner joiner = pluginInstantiator.newPluginInstance(stageName, evaluator); ContextProvider<DefaultJoinerContext> contextProvider = new JoinerContextProvider(pipelineRuntime, stageSpec, admin);
private <IN, ERROR> TrackedMultiOutputTransform<IN, ERROR> getMultiOutputTransform(StageSpec stageSpec) throws Exception { String stageName = stageSpec.getName(); DefaultMacroEvaluator macroEvaluator = new DefaultMacroEvaluator(arguments, taskContext.getLogicalStartTime(), taskContext, taskContext.getNamespace()); SplitterTransform<IN, ERROR> splitterTransform = pluginInstantiator.newPluginInstance(stageName, macroEvaluator); TransformContext transformContext = createRuntimeContext(stageSpec); splitterTransform.initialize(transformContext); StageMetrics stageMetrics = new DefaultStageMetrics(metrics, stageName); TaskAttemptContext taskAttemptContext = (TaskAttemptContext) taskContext.getHadoopContext(); StageStatisticsCollector collector = isPipelineContainsCondition ? new MapReduceStageStatisticsCollector(stageName, taskAttemptContext) : new NoopStageStatisticsCollector(); return new TrackedMultiOutputTransform<>(splitterTransform, stageMetrics, taskContext.getDataTracer(stageName), collector); }
new MapReduceStageStatisticsCollector(stageName, taskAttemptContext) : new NoopStageStatisticsCollector(); if (BatchAggregator.PLUGIN_TYPE.equals(pluginType)) { BatchAggregator<?, ?, ?> batchAggregator = pluginInstantiator.newPluginInstance(stageName, macroEvaluator); BatchRuntimeContext runtimeContext = createRuntimeContext(stageSpec); batchAggregator.initialize(runtimeContext); BatchJoiner<?, ?, ?> batchJoiner = pluginInstantiator.newPluginInstance(stageName, macroEvaluator); BatchJoinerRuntimeContext runtimeContext = createRuntimeContext(stageSpec); batchJoiner.initialize(runtimeContext);