@SuppressWarnings("unchecked") private void destroyWorkflow() { if (!(workflow instanceof ProgramLifecycle)) { return; } final TransactionControl txControl = Transactions.getTransactionControl(workflowContext.getDefaultTxControl(), Workflow.class, workflow, "destroy"); basicWorkflowToken.setCurrentNode(workflowSpec.getName()); workflowContext.destroyProgram((ProgramLifecycle) workflow, txControl, false); try { workflowStateWriter.setWorkflowToken(workflowRunId, basicWorkflowToken); } catch (Throwable t) { LOG.error("Failed to store the final workflow token of Workflow {}", workflowRunId, t); } if (ProgramStatus.COMPLETED != workflowContext.getState().getStatus()) { return; } try { Set<Operation> fieldLineageOperations = workflowContext.getFieldLineageOperations(); if (!fieldLineageOperations.isEmpty()) { FieldLineageInfo info = new FieldLineageInfo(fieldLineageOperations); fieldLineageWriter.write(workflowRunId, info); } } catch (Throwable t) { LOG.debug("Failed to emit the field lineage operations for Workflow {}", workflowRunId, t); } }
private boolean keepLocal(String originalDatasetName) { Map<String, String> datasetArguments = RuntimeArguments.extractScope(Scope.DATASET, originalDatasetName, workflowContext.getRuntimeArguments()); return Boolean.parseBoolean(datasetArguments.get("keep.local")); }
WorkflowToken token) throws Exception { final BasicWorkflowContext context = new BasicWorkflowContext(workflowSpec, token, program, programOptions, cConf, metricsCollectionService, datasetFramework, txClient, discoveryServiceClient, nodeStates, new DataSetFieldSetter(context), new MetricsFieldSetter(context.getMetrics())); TransactionControl defaultTxControl = workflowContext.getDefaultTxControl(); try { workflowCondition, "initialize", WorkflowContext.class); context.initializeProgram(workflowCondition, txControl, false); boolean result = context.execute(() -> workflowCondition.apply(context)); iterator = result ? node.getIfBranch().iterator() : node.getElseBranch().iterator(); } finally { TransactionControl txControl = Transactions.getTransactionControl(defaultTxControl, Condition.class, workflowCondition, "destroy"); context.destroyProgram(workflowCondition, txControl, false);
@SuppressWarnings("unchecked") private Workflow initializeWorkflow() throws Exception { Class<?> clz = Class.forName(workflowSpec.getClassName(), true, program.getClassLoader()); if (!Workflow.class.isAssignableFrom(clz)) { throw new IllegalStateException(String.format("%s is not Workflow.", clz)); } Class<? extends Workflow> workflowClass = (Class<? extends Workflow>) clz; final Workflow workflow = new InstantiatorFactory(false).get(TypeToken.of(workflowClass)).create(); // set metrics Reflections.visit(workflow, workflow.getClass(), new MetricsFieldSetter(workflowContext.getMetrics())); if (!(workflow instanceof ProgramLifecycle)) { return workflow; } final TransactionControl txControl = Transactions.getTransactionControl(workflowContext.getDefaultTxControl(), Workflow.class, workflow, "initialize", WorkflowContext.class); basicWorkflowToken.setCurrentNode(workflowSpec.getName()); workflowContext.setState(new ProgramState(ProgramStatus.INITIALIZING, null)); workflowContext.initializeProgram((ProgramLifecycle) workflow, txControl, false); workflowStateWriter.setWorkflowToken(workflowRunId, basicWorkflowToken); return workflow; }
workflowRunId.getRun(), node.getNodeId(), (BasicWorkflowToken) token, workflowContext.fieldLineageConsolidationEnabled()); ProgramOptions actionOptions = new SimpleProgramOptions(programOptions.getProgramId(),
WorkflowToken token) throws Exception { final BasicWorkflowContext context = new BasicWorkflowContext(workflowSpec, token, program, programOptions, cConf, metricsCollectionService, datasetFramework, txClient, discoveryServiceClient, nodeStates, new DataSetFieldSetter(context), new MetricsFieldSetter(context.getMetrics())); TransactionControl defaultTxControl = workflowContext.getDefaultTxControl(); try { workflowCondition, "initialize", WorkflowContext.class); context.initializeProgram(workflowCondition, txControl, false); boolean result = context.execute(() -> workflowCondition.apply(context)); iterator = result ? node.getIfBranch().iterator() : node.getElseBranch().iterator(); } finally { TransactionControl txControl = Transactions.getTransactionControl(defaultTxControl, Condition.class, workflowCondition, "destroy"); context.destroyProgram(workflowCondition, txControl, false);
@SuppressWarnings("unchecked") private Workflow initializeWorkflow() throws Exception { Class<?> clz = Class.forName(workflowSpec.getClassName(), true, program.getClassLoader()); if (!Workflow.class.isAssignableFrom(clz)) { throw new IllegalStateException(String.format("%s is not Workflow.", clz)); } Class<? extends Workflow> workflowClass = (Class<? extends Workflow>) clz; final Workflow workflow = new InstantiatorFactory(false).get(TypeToken.of(workflowClass)).create(); // set metrics Reflections.visit(workflow, workflow.getClass(), new MetricsFieldSetter(workflowContext.getMetrics())); if (!(workflow instanceof ProgramLifecycle)) { return workflow; } final TransactionControl txControl = Transactions.getTransactionControl(workflowContext.getDefaultTxControl(), Workflow.class, workflow, "initialize", WorkflowContext.class); basicWorkflowToken.setCurrentNode(workflowSpec.getName()); workflowContext.setState(new ProgramState(ProgramStatus.INITIALIZING, null)); workflowContext.initializeProgram((ProgramLifecycle) workflow, txControl, false); workflowStateWriter.setWorkflowToken(workflowRunId, basicWorkflowToken); return workflow; }
workflowRunId.getRun(), node.getNodeId(), (BasicWorkflowToken) token, workflowContext.fieldLineageConsolidationEnabled()); ProgramOptions actionOptions = new SimpleProgramOptions(programOptions.getProgramId(),
@SuppressWarnings("unchecked") private void destroyWorkflow() { if (!(workflow instanceof ProgramLifecycle)) { return; } final TransactionControl txControl = Transactions.getTransactionControl(workflowContext.getDefaultTxControl(), Workflow.class, workflow, "destroy"); basicWorkflowToken.setCurrentNode(workflowSpec.getName()); workflowContext.destroyProgram((ProgramLifecycle) workflow, txControl, false); try { workflowStateWriter.setWorkflowToken(workflowRunId, basicWorkflowToken); } catch (Throwable t) { LOG.error("Failed to store the final workflow token of Workflow {}", workflowRunId, t); } if (ProgramStatus.COMPLETED != workflowContext.getState().getStatus()) { return; } try { Set<Operation> fieldLineageOperations = workflowContext.getFieldLineageOperations(); if (!fieldLineageOperations.isEmpty()) { FieldLineageInfo info = new FieldLineageInfo(fieldLineageOperations); fieldLineageWriter.write(workflowRunId, info); } } catch (Throwable t) { LOG.debug("Failed to emit the field lineage operations for Workflow {}", workflowRunId, t); } }
private boolean keepLocal(String originalDatasetName) { Map<String, String> datasetArguments = RuntimeArguments.extractScope(Scope.DATASET, originalDatasetName, workflowContext.getRuntimeArguments()); return Boolean.parseBoolean(datasetArguments.get("keep.local")); }