public JobSpecification compilePlan(ILogicalPlan plan, IJobletEventListenerFactory jobEventListenerFactory) throws AlgebricksException { return compilePlanImpl(plan, false, null, jobEventListenerFactory); }
throw new NotImplementedException("Nested plans with several roots are not supported."); JobSpecification nestedJob = pc.compileNestedPlan(p, outerPlanSchema); ILogicalOperator topOpInSubplan = p.getRoots().get(0).getValue(); JobGenContext context = pc.getContext(); IOperatorSchema topOpInSubplanScm = context.getSchema(topOpInSubplan); opSchema.addAllVariables(topOpInSubplanScm);
private JobSpecification compilePlanImpl(ILogicalPlan plan, boolean isNestedPlan, IOperatorSchema outerPlanSchema, IJobletEventListenerFactory jobEventListenerFactory) throws AlgebricksException { JobSpecification spec = new JobSpecification(context.getFrameSize()); if (jobEventListenerFactory != null) { spec.setJobletEventListenerFactory(jobEventListenerFactory); } List<ILogicalOperator> rootOps = new ArrayList<>(); JobBuilder builder = new JobBuilder(spec, context.getClusterLocations()); for (Mutable<ILogicalOperator> opRef : plan.getRoots()) { compileOpRef(opRef, spec, builder, outerPlanSchema); rootOps.add(opRef.getValue()); } reviseEdges(builder); operatorVisitedToParents.clear(); builder.buildSpec(rootOps); spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy()); // Do not do activity cluster planning because it is slow on large clusters spec.setUseConnectorPolicyForScheduling(false); if (isNestedPlan) { spec.setMetaOps(builder.getGeneratedMetaOps()); } return spec; }
protected List<List<AlgebricksPipeline>> compileSubplansImpl(IOperatorSchema outerPlanSchema, AbstractOperatorWithNestedPlans npOp, IOperatorSchema opSchema, JobGenContext context) throws AlgebricksException { List<List<AlgebricksPipeline>> subplans = new ArrayList<>(npOp.getNestedPlans().size()); PlanCompiler pc = new PlanCompiler(context); for (ILogicalPlan p : npOp.getNestedPlans()) { subplans.add(buildPipelineWithProjection(p, outerPlanSchema, npOp, opSchema, pc)); } return subplans; }
private void compileOpRef(Mutable<ILogicalOperator> opRef, IOperatorDescriptorRegistry spec, IHyracksJobBuilder builder, IOperatorSchema outerPlanSchema) throws AlgebricksException { ILogicalOperator op = opRef.getValue(); int n = op.getInputs().size(); IOperatorSchema[] schemas = new IOperatorSchema[n]; int i = 0; for (Mutable<ILogicalOperator> opChild : op.getInputs()) { List<Mutable<ILogicalOperator>> parents = operatorVisitedToParents.get(opChild); if (parents == null) { parents = new ArrayList<Mutable<ILogicalOperator>>(); operatorVisitedToParents.put(opChild, parents); parents.add(opRef); compileOpRef(opChild, spec, builder, outerPlanSchema); schemas[i++] = context.getSchema(opChild.getValue()); } else { if (!parents.contains(opRef)) parents.add(opRef); schemas[i++] = context.getSchema(opChild.getValue()); continue; } } IOperatorSchema opSchema = new OperatorSchemaImpl(); context.putSchema(op, opSchema); op.getVariablePropagationPolicy().propagateVariables(opSchema, schemas); op.contributeRuntimeOperator(builder, context, opSchema, schemas, outerPlanSchema); }
public JobSpecification compileNestedPlan(ILogicalPlan plan, IOperatorSchema outerPlanSchema) throws AlgebricksException { return compilePlanImpl(plan, true, outerPlanSchema, null); }