public static IFrameWriter assemblePipeline(AlgebricksPipeline subplan, IFrameWriter writer, IHyracksTaskContext ctx) throws HyracksDataException { // should enforce protocol boolean enforce = ctx.getJobFlags().contains(JobFlag.ENFORCE_CONTRACT); // plug the operators IFrameWriter start = writer; IPushRuntimeFactory[] runtimeFactories = subplan.getRuntimeFactories(); RecordDescriptor[] recordDescriptors = subplan.getRecordDescriptors(); for (int i = runtimeFactories.length - 1; i >= 0; i--) { IPushRuntime newRuntime = runtimeFactories[i].createPushRuntime(ctx)[0]; newRuntime = enforce ? EnforcePushRuntime.enforce(newRuntime) : newRuntime; start = enforce ? EnforceFrameWriter.enforce(start) : start; newRuntime.setOutputFrameWriter(0, start, recordDescriptors[i]); if (i > 0) { newRuntime.setInputRecordDescriptor(0, recordDescriptors[i - 1]); } else { // the nts has the same input and output rec. desc. newRuntime.setInputRecordDescriptor(0, recordDescriptors[0]); } start = newRuntime; } return start; } }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider(); RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); int[] primaryKeyFields = JobGenHelper.variablesToFieldIndexes(primaryKeyLogicalVars, inputSchemas[0]); //get dataset splits IPushRuntimeFactory runtime = dataset.getCommitRuntimeFactory(metadataProvider, primaryKeyFields, isSink); runtime.setSourceLocation(op.getSourceLocation()); builder.contributeMicroOperator(op, runtime, recDesc); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); }
IPushRuntime[] newRuntimes = runtimeFactory.createPushRuntime(ctx); for (int j = 0; j < newRuntimes.length; j++) { if (enforce) {
mp.getWriteFileRuntime(write.getDataSink(), columns, pf, inputDesc); IPushRuntimeFactory runtime = runtimeAndConstraints.first; runtime.setSourceLocation(write.getSourceLocation());
.createPushRuntime(ctx)[0]; IPushRuntime commitOp = dataset.getCommitRuntimeFactory(mdProvider, primaryIndexInfo.primaryKeyIndexes, true) .createPushRuntime(ctx)[0]; insertOp.setOutputFrameWriter(0, commitOp, primaryIndexInfo.rDesc); commitOp.setInputRecordDescriptor(0, primaryIndexInfo.rDesc);