@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); MicroUnionAllRuntimeFactory runtime = new MicroUnionAllRuntimeFactory(op.getInputs().size()); runtime.setSourceLocation(op.getSourceLocation()); builder.contributeMicroOperator(op, runtime, recordDescriptor); super.contributeRuntimeOperator(builder, context, op, opSchema, inputSchemas, outerPlanSchema); } }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { int keys[] = JobGenHelper.variablesToFieldIndexes(columnList, inputSchemas[0]); GroupByOperator gby = (GroupByOperator) op; IVariableTypeEnvironment env = context.getTypeEnvironment(op); int fdColumns[] = getFdColumns(gby, inputSchemas[0]); // compile subplans and set the gby op. schema accordingly AlgebricksPipeline[] subplans = compileSubplans(inputSchemas[0], gby, opSchema, context); IAggregatorDescriptorFactory aggregatorFactory = new NestedPlansAccumulatingAggregatorFactory(subplans, keys, fdColumns); IBinaryComparatorFactory[] comparatorFactories = JobGenHelper.variablesToAscBinaryComparatorFactories(columnList, env, context); RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); RecordDescriptor inputRecordDesc = JobGenHelper.mkRecordDescriptor( context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context); MicroPreClusteredGroupRuntimeFactory runtime = new MicroPreClusteredGroupRuntimeFactory(keys, comparatorFactories, aggregatorFactory, inputRecordDesc, recordDescriptor, null); runtime.setSourceLocation(gby.getSourceLocation()); builder.contributeMicroOperator(gby, runtime, recordDescriptor); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { RunningAggregateOperator ragg = (RunningAggregateOperator) op; List<LogicalVariable> variables = ragg.getVariables(); List<Mutable<ILogicalExpression>> expressions = ragg.getExpressions(); int[] outColumns = new int[variables.size()]; for (int i = 0; i < outColumns.length; i++) { outColumns[i] = opSchema.findVariable(variables.get(i)); } IRunningAggregateEvaluatorFactory[] runningAggFuns = new IRunningAggregateEvaluatorFactory[expressions.size()]; IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider(); for (int i = 0; i < runningAggFuns.length; i++) { StatefulFunctionCallExpression expr = (StatefulFunctionCallExpression) expressions.get(i).getValue(); runningAggFuns[i] = expressionRuntimeProvider.createRunningAggregateFunctionFactory(expr, context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas, context); } // TODO push projections into the operator int[] projectionList = JobGenHelper.projectAllVariables(opSchema); RunningAggregateRuntimeFactory runtime = new RunningAggregateRuntimeFactory(projectionList, outColumns, runningAggFuns); runtime.setSourceLocation(ragg.getSourceLocation()); // contribute one Asterix framewriter RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); builder.contributeMicroOperator(ragg, runtime, recDesc); // and contribute one edge from its child ILogicalOperator src = ragg.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, ragg, 0); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider(); RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); int[] primaryKeyFields = JobGenHelper.variablesToFieldIndexes(primaryKeyLogicalVars, inputSchemas[0]); //get dataset splits IPushRuntimeFactory runtime = dataset.getCommitRuntimeFactory(metadataProvider, primaryKeyFields, isSink); runtime.setSourceLocation(op.getSourceLocation()); builder.contributeMicroOperator(op, runtime, recDesc); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); }
IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { int[] keysLeft = JobGenHelper.variablesToFieldIndexes(keysLeftBranch, inputSchemas[0]); int[] keysRight = JobGenHelper.variablesToFieldIndexes(keysRightBranch, inputSchemas[1]); IVariableTypeEnvironment env = context.getTypeEnvironment(op); IBinaryHashFunctionFamily[] hashFunFamilies = JobGenHelper.variablesToBinaryHashFunctionFamilies(keysLeftBranch, env, context); IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[keysLeft.length]; int i = 0; JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); IOperatorDescriptorRegistry spec = builder.getJobSpec(); IOperatorDescriptor opDesc;
throws AlgebricksException { List<LogicalVariable> gbyCols = getGbyColumns(); int keys[] = JobGenHelper.variablesToFieldIndexes(gbyCols, inputSchemas[0]); GroupByOperator gby = (GroupByOperator) op; int numFds = gby.getDecorList().size(); JobGenHelper.variablesToAscBinaryComparatorFactories(gbyCols, aggOpInputEnv, context); RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); IBinaryHashFunctionFamily[] hashFunctionFactories = JobGenHelper.variablesToBinaryHashFunctionFamilies(gbyCols, aggOpInputEnv, context); JobGenHelper.variablesToAscNormalizedKeyComputerFactory(gbyCols, aggOpInputEnv, context);
WindowOperator winOp = (WindowOperator) op; int[] partitionColumnsList = JobGenHelper.projectVariables(inputSchemas[0], partitionColumns); JobGenHelper.variablesToAscBinaryComparatorFactories(partitionColumns, opTypeEnv, context); JobGenHelper.variablesToBinaryComparatorFactories(orderColumns, opTypeEnv, context); int[] projectionColumnsExcludingSubplans = JobGenHelper.projectAllVariables(opSchema); int[] runningAggOutColumns = JobGenHelper.projectVariables(opSchema, winOp.getVariables()); RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(opTypeEnv, opSchema, context); builder.contributeMicroOperator(winOp, runtime, recDesc);
IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { int[] keysLeft = JobGenHelper.variablesToFieldIndexes(keysLeftBranch, inputSchemas[0]); int[] keysRight = JobGenHelper.variablesToFieldIndexes(keysRightBranch, inputSchemas[1]); IVariableTypeEnvironment env = context.getTypeEnvironment(op); IBinaryHashFunctionFactory[] hashFunFactories = JobGenHelper.variablesToBinaryHashFunctionFactories(keysLeftBranch, env, context); IBinaryComparatorFactory[] comparatorFactories = new IBinaryComparatorFactory[keysLeft.length]; int i = 0; JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); IOperatorDescriptorRegistry spec = builder.getJobSpec(); IOperatorDescriptor opDesc;
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { int[] keysAndDecs = getKeysAndDecs(inputSchemas[0]); IBinaryComparatorFactory[] comparatorFactories = JobGenHelper .variablesToAscBinaryComparatorFactories(columnList, context.getTypeEnvironment(op), context); IAggregateEvaluatorFactory[] aggFactories = new IAggregateEvaluatorFactory[] {}; AbstractAggregatorDescriptorFactory aggregatorFactory = new SimpleAlgebricksAccumulatingAggregatorFactory(aggFactories, keysAndDecs); aggregatorFactory.setSourceLocation(op.getSourceLocation()); RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); RecordDescriptor inputRecordDesc = JobGenHelper.mkRecordDescriptor( context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context); /* make fd columns part of the key but the comparator only compares the distinct key columns */ MicroPreClusteredGroupRuntimeFactory runtime = new MicroPreClusteredGroupRuntimeFactory(keysAndDecs, comparatorFactories, aggregatorFactory, inputRecordDesc, recordDescriptor, null); runtime.setSourceLocation(op.getSourceLocation()); builder.contributeMicroOperator(op, runtime, recordDescriptor); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); } }
JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor( context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context); JobGenHelper.mkPrinterFactories(inputSchemas[0], context.getTypeEnvironment(op), context, columns);
protected int[] getKeysAndDecs(IOperatorSchema inputSchema) { int keys[] = JobGenHelper.variablesToFieldIndexes(columnList, inputSchema); int sz = inputSchema.getSize(); int fdSz = sz - columnList.size(); int[] fdColumns = new int[fdSz]; int j = 0; for (LogicalVariable v : inputSchema) { if (!columnList.contains(v)) { fdColumns[j++] = inputSchema.findVariable(v); } } int[] keysAndDecs = new int[keys.length + fdColumns.length]; for (int i = 0; i < keys.length; i++) { keysAndDecs[i] = keys[i]; } for (int i = 0; i < fdColumns.length; i++) { keysAndDecs[i + keys.length] = fdColumns[i]; } return keysAndDecs; }
public static int[] projectVariables(IOperatorSchema opSchema, List<LogicalVariable> variables) { return projectVariablesImpl(opSchema, variables, variables.size()); }
throws AlgebricksException { List<LogicalVariable> gbyCols = getGbyColumns(); int keys[] = JobGenHelper.variablesToFieldIndexes(gbyCols, inputSchemas[0]); GroupByOperator gby = (GroupByOperator) op; int numFds = gby.getDecorList().size(); JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), localInputSchemas[0], context);
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { IOperatorDescriptorRegistry spec = builder.getJobSpec(); int[] keysAndDecs = getKeysAndDecs(inputSchemas[0]); IBinaryComparatorFactory[] comparatorFactories = JobGenHelper .variablesToAscBinaryComparatorFactories(columnList, context.getTypeEnvironment(op), context); IAggregateEvaluatorFactory[] aggFactories = new IAggregateEvaluatorFactory[] {}; AbstractAggregatorDescriptorFactory aggregatorFactory = new SimpleAlgebricksAccumulatingAggregatorFactory(aggFactories, keysAndDecs); aggregatorFactory.setSourceLocation(op.getSourceLocation()); RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); /* make fd columns part of the key but the comparator only compares the distinct key columns */ PreclusteredGroupOperatorDescriptor opDesc = new PreclusteredGroupOperatorDescriptor(spec, keysAndDecs, comparatorFactories, aggregatorFactory, recordDescriptor); opDesc.setSourceLocation(op.getSourceLocation()); contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); } }
columns[i++] = inputSchemas[0].findVariable(v); RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor( context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context); JobGenHelper.mkPrinterFactories(inputSchemas[0], context.getTypeEnvironment(op), context, columns);
public static int[] projectAllVariables(IOperatorSchema opSchema) { return projectVariablesImpl(opSchema, opSchema, opSchema.getSize()); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { SubplanOperator subplan = (SubplanOperator) op; if (subplan.getNestedPlans().size() != 1) { throw new NotImplementedException("Subplan currently works only for one nested plan with one root."); } List<List<AlgebricksPipeline>> subplans = compileSubplansImpl(inputSchemas[0], subplan, opSchema, context); assert subplans.size() == 1; List<AlgebricksPipeline> np = subplans.get(0); RecordDescriptor inputRecordDesc = JobGenHelper.mkRecordDescriptor( context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context); IMissingWriterFactory[] missingWriterFactories = new IMissingWriterFactory[np.get(0).getOutputWidth()]; for (int i = 0; i < missingWriterFactories.length; i++) { missingWriterFactories[i] = context.getMissingWriterFactory(); } RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); SubplanRuntimeFactory runtime = new SubplanRuntimeFactory(np, missingWriterFactories, inputRecordDesc, recDesc, null); runtime.setSourceLocation(subplan.getSourceLocation()); builder.contributeMicroOperator(subplan, runtime, recDesc); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { int keys[] = JobGenHelper.variablesToFieldIndexes(columnList, inputSchemas[0]); GroupByOperator gby = (GroupByOperator) op; int fdColumns[] = getFdColumns(gby, inputSchemas[0]); // compile subplans and set the gby op. schema accordingly AlgebricksPipeline[] subplans = compileSubplans(inputSchemas[0], gby, opSchema, context); AbstractAggregatorDescriptorFactory aggregatorFactory; List<ILogicalPlan> nestedPlans = gby.getNestedPlans(); if (!nestedPlans.isEmpty() && nestedPlans.get(0).getRoots().get(0).getValue() .getOperatorTag() == LogicalOperatorTag.RUNNINGAGGREGATE) { aggregatorFactory = new NestedPlansRunningAggregatorFactory(subplans, keys, fdColumns); } else { aggregatorFactory = new NestedPlansAccumulatingAggregatorFactory(subplans, keys, fdColumns); } aggregatorFactory.setSourceLocation(gby.getSourceLocation()); IOperatorDescriptorRegistry spec = builder.getJobSpec(); IBinaryComparatorFactory[] comparatorFactories = JobGenHelper .variablesToAscBinaryComparatorFactories(columnList, context.getTypeEnvironment(op), context); RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); PreclusteredGroupOperatorDescriptor opDesc = new PreclusteredGroupOperatorDescriptor(spec, keys, comparatorFactories, aggregatorFactory, recordDescriptor, groupAll, framesLimit); opDesc.setSourceLocation(gby.getSourceLocation()); contributeOpDesc(builder, gby, opDesc); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); }
int[] projectionList = JobGenHelper.projectAllVariables(opSchema); RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); if (locations != null && locations.length > 0) { AlgebricksAbsolutePartitionConstraint locationConstraint =
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { propagatedSchema.addAllVariables(outerPlanSchema); NestedTupleSourceRuntimeFactory runtime = new NestedTupleSourceRuntimeFactory(); runtime.setSourceLocation(op.getSourceLocation()); RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); builder.contributeMicroOperator(op, runtime, recDesc); }