public JobSpecification compilePlan(ILogicalPlan plan, IJobletEventListenerFactory jobEventListenerFactory) throws AlgebricksException { return compilePlanImpl(plan, false, null, jobEventListenerFactory); }
@Override public void contributeMicroOperator(ILogicalOperator op, IPushRuntimeFactory runtime, RecordDescriptor recDesc) { contributeMicroOperator(op, runtime, recDesc, null); }
@Override public void buildSpec(List<ILogicalOperator> roots) throws AlgebricksException { buildAsterixComponents(); Map<IConnectorDescriptor, TargetConstraint> tgtConstraints = setupConnectors(); for (ILogicalOperator r : roots) { IOperatorDescriptor opDesc = findOpDescForAlgebraicOp(r); jobSpec.addRoot(opDesc); } setAllPartitionConstraints(tgtConstraints); }
private JobSpecification compilePlanImpl(ILogicalPlan plan, boolean isNestedPlan, IOperatorSchema outerPlanSchema, IJobletEventListenerFactory jobEventListenerFactory) throws AlgebricksException { JobSpecification spec = new JobSpecification(context.getFrameSize()); if (jobEventListenerFactory != null) { spec.setJobletEventListenerFactory(jobEventListenerFactory); } List<ILogicalOperator> rootOps = new ArrayList<>(); JobBuilder builder = new JobBuilder(spec, context.getClusterLocations()); for (Mutable<ILogicalOperator> opRef : plan.getRoots()) { compileOpRef(opRef, spec, builder, outerPlanSchema); rootOps.add(opRef.getValue()); } reviseEdges(builder); operatorVisitedToParents.clear(); builder.buildSpec(rootOps); spec.setConnectorPolicyAssignmentPolicy(new ConnectorPolicyAssignmentPolicy()); // Do not do activity cluster planning because it is slow on large clusters spec.setUseConnectorPolicyForScheduling(false); if (isNestedPlan) { spec.setMetaOps(builder.getGeneratedMetaOps()); } return spec; }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { propagatedSchema.addAllVariables(outerPlanSchema); NestedTupleSourceRuntimeFactory runtime = new NestedTupleSourceRuntimeFactory(); runtime.setSourceLocation(op.getSourceLocation()); RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); builder.contributeMicroOperator(op, runtime, recDesc); }
@Override public AsterixTupleFilterFactory createTupleFilterFactory(IOperatorSchema[] inputSchemas, IVariableTypeEnvironment typeEnv, ILogicalExpression filterExpr, JobGenContext context) throws AlgebricksException { // No filtering condition. if (filterExpr == null) { return null; } IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider(); IScalarEvaluatorFactory filterEvalFactory = expressionRuntimeProvider.createEvaluatorFactory(filterExpr, typeEnv, inputSchemas, context); return new AsterixTupleFilterFactory(filterEvalFactory, context.getBinaryBooleanInspectorFactory()); }
private void setAllPartitionConstraints(Map<IConnectorDescriptor, TargetConstraint> tgtConstraints) throws AlgebricksException { List<OperatorDescriptorId> roots = jobSpec.getRoots(); setSpecifiedPartitionConstraints(); for (OperatorDescriptorId rootId : roots) { setPartitionConstraintsBottomup(rootId, tgtConstraints, null, false); } for (OperatorDescriptorId rootId : roots) { setPartitionConstraintsTopdown(rootId, tgtConstraints, null); } for (OperatorDescriptorId rootId : roots) { setPartitionConstraintsBottomup(rootId, tgtConstraints, null, true); } }
private void buildAsterixComponents() { for (ILogicalOperator aop : microOps.keySet()) { addMicroOpToMetaRuntimeOp(aop); } for (Integer k : metaAsterixOpSkeletons.keySet()) { List<Pair<IPushRuntimeFactory, RecordDescriptor>> opContents = metaAsterixOpSkeletons.get(k); AlgebricksMetaOperatorDescriptor amod = buildMetaAsterixOpDesc(opContents); metaAsterixOps.put(k, amod); } }
public static int[] projectVariables(IOperatorSchema opSchema, List<LogicalVariable> variables) { return projectVariablesImpl(opSchema, variables, variables.size()); }
public static IBinaryComparatorFactory[] variablesToAscBinaryComparatorFactories(List<LogicalVariable> varLogical, int start, int size, IVariableTypeEnvironment env, JobGenContext context) throws AlgebricksException { IBinaryComparatorFactory[] compFactories = new IBinaryComparatorFactory[size]; IBinaryComparatorFactoryProvider bcfProvider = context.getBinaryComparatorFactoryProvider(); for (int i = 0; i < size; i++) { Object type = env.getVarType(varLogical.get(start + i)); compFactories[i] = bcfProvider.getBinaryComparatorFactory(type, true); } return compFactories; }
public static ITypeTraits[] variablesToTypeTraits(Collection<LogicalVariable> varLogical, IVariableTypeEnvironment env, JobGenContext context) throws AlgebricksException { ITypeTraits[] typeTraits = new ITypeTraits[varLogical.size()]; ITypeTraitProvider typeTraitProvider = context.getTypeTraitProvider(); int i = 0; for (LogicalVariable v : varLogical) { Object type = env.getVarType(v); typeTraits[i++] = typeTraitProvider.getTypeTrait(type); } return typeTraits; }
@Override public void contributeGraphEdge(ILogicalOperator src, int srcOutputIndex, ILogicalOperator dest, int destInputIndex) { ArrayList<ILogicalOperator> outputs = outEdges.get(src); if (outputs == null) { outputs = new ArrayList<>(); outEdges.put(src, outputs); } addAtPos(outputs, dest, srcOutputIndex); ArrayList<ILogicalOperator> inp = inEdges.get(dest); if (inp == null) { inp = new ArrayList<>(); inEdges.put(dest, inp); } addAtPos(inp, src, destInputIndex); }
public static IBinaryHashFunctionFamily[] variablesToBinaryHashFunctionFamilies( Collection<LogicalVariable> varLogical, IVariableTypeEnvironment env, JobGenContext context) throws AlgebricksException { IBinaryHashFunctionFamily[] funFamilies = new IBinaryHashFunctionFamily[varLogical.size()]; int i = 0; IBinaryHashFunctionFamilyProvider bhffProvider = context.getBinaryHashFunctionFamilyProvider(); for (LogicalVariable var : varLogical) { Object type = env.getVarType(var); funFamilies[i++] = bhffProvider.getBinaryHashFunctionFamily(type); } return funFamilies; }
public static INormalizedKeyComputerFactory variablesToAscNormalizedKeyComputerFactory( Collection<LogicalVariable> varLogical, IVariableTypeEnvironment env, JobGenContext context) throws AlgebricksException { INormalizedKeyComputerFactoryProvider nkcfProvider = context.getNormalizedKeyComputerFactoryProvider(); if (nkcfProvider == null) return null; for (LogicalVariable v : varLogical) { Object type = env.getVarType(v); return nkcfProvider.getNormalizedKeyComputerFactory(type, true); } return null; }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); MicroUnionAllRuntimeFactory runtime = new MicroUnionAllRuntimeFactory(op.getInputs().size()); runtime.setSourceLocation(op.getSourceLocation()); builder.contributeMicroOperator(op, runtime, recordDescriptor); super.contributeRuntimeOperator(builder, context, op, opSchema, inputSchemas, outerPlanSchema); } }
public JobSpecification compileNestedPlan(ILogicalPlan plan, IOperatorSchema outerPlanSchema) throws AlgebricksException { return compilePlanImpl(plan, true, outerPlanSchema, null); }
public static int[] projectAllVariables(IOperatorSchema opSchema) { return projectVariablesImpl(opSchema, opSchema, opSchema.getSize()); }
public static IBinaryComparatorFactory[] variablesToAscBinaryComparatorFactories( Collection<LogicalVariable> varLogical, IVariableTypeEnvironment env, JobGenContext context) throws AlgebricksException { IBinaryComparatorFactory[] compFactories = new IBinaryComparatorFactory[varLogical.size()]; IBinaryComparatorFactoryProvider bcfProvider = context.getBinaryComparatorFactoryProvider(); int i = 0; for (LogicalVariable v : varLogical) { Object type = env.getVarType(v); compFactories[i++] = bcfProvider.getBinaryComparatorFactory(type, true); } return compFactories; }
public static ITypeTraits[] variablesToTypeTraits(List<LogicalVariable> varLogical, int start, int size, IVariableTypeEnvironment env, JobGenContext context) throws AlgebricksException { ITypeTraits[] typeTraits = new ITypeTraits[size]; ITypeTraitProvider typeTraitProvider = context.getTypeTraitProvider(); for (int i = 0; i < size; i++) { Object type = env.getVarType(varLogical.get(start + i)); typeTraits[i] = typeTraitProvider.getTypeTrait(type); } return typeTraits; }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); MaterializingOperatorDescriptor materializationOpDesc = new MaterializingOperatorDescriptor(builder.getJobSpec(), recDescriptor, isSingleActivity); materializationOpDesc.setSourceLocation(op.getSourceLocation()); contributeOpDesc(builder, (AbstractLogicalOperator) op, materializationOpDesc); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); }