@SuppressWarnings({ "rawtypes", "unchecked" }) @Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { WriteResultOperator writeResultOp = (WriteResultOperator) op; IMetadataProvider mp = context.getMetadataProvider(); JobSpecification spec = builder.getJobSpec(); Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getWriteResultRuntime( dataSource, propagatedSchema, keys, payload, additionalFilteringKeys, context, spec); IOperatorDescriptor opDesc = runtimeAndConstraints.first; opDesc.setSourceLocation(writeResultOp.getSourceLocation()); builder.contributeHyracksOperator(writeResultOp, opDesc); builder.contributeAlgebricksPartitionConstraint(opDesc, runtimeAndConstraints.second); ILogicalOperator src = writeResultOp.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, writeResultOp, 0); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider(); RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); int[] primaryKeyFields = JobGenHelper.variablesToFieldIndexes(primaryKeyLogicalVars, inputSchemas[0]); //get dataset splits IPushRuntimeFactory runtime = dataset.getCommitRuntimeFactory(metadataProvider, primaryKeyFields, isSink); runtime.setSourceLocation(op.getSourceLocation()); builder.contributeMicroOperator(op, runtime, recDesc); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { Pair<IConnectorDescriptor, TargetConstraint> connPair = createConnectorDescriptor(builder.getJobSpec(), op, opSchema, context); builder.contributeConnectorWithTargetConstraint(op, connPair.first, connPair.second); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); }
protected void contributeOpDesc(IHyracksJobBuilder builder, AbstractLogicalOperator op, IOperatorDescriptor opDesc) { if (op.getExecutionMode() == ExecutionMode.UNPARTITIONED) { AlgebricksPartitionConstraint apc = new AlgebricksCountPartitionConstraint(1); builder.contributeAlgebricksPartitionConstraint(opDesc, apc); } builder.contributeHyracksOperator(op, opDesc); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { IOperatorDescriptorRegistry spec = builder.getJobSpec(); SinkOperatorDescriptor opDesc = new SinkOperatorDescriptor(spec, op.getInputs().size()); opDesc.setSourceLocation(op.getSourceLocation()); contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc); for (int i = 0; i < op.getInputs().size(); i++) { builder.contributeGraphEdge(op.getInputs().get(i).getValue(), 0, op, i); } }
/** * The output record descriptor of forward operator is the same as the output record descriptor of the data source * which is located at index 0. * @param builder Hyracks job builder * @param context job generation context * @param op {@see {@link ForwardOperator}} * @param propagatedSchema not used * @param inputSchemas schemas of all inputs * @param outerPlanSchema not used * @throws AlgebricksException */ @Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { ForwardOperator forwardOp = (ForwardOperator) op; RecordDescriptor dataInputDescriptor = JobGenHelper.mkRecordDescriptor( context.getTypeEnvironment(forwardOp.getInputs().get(0).getValue()), inputSchemas[0], context); ForwardOperatorDescriptor forwardDescriptor = new ForwardOperatorDescriptor(builder.getJobSpec(), forwardOp.getRangeMapKey(), dataInputDescriptor); builder.contributeHyracksOperator(forwardOp, forwardDescriptor); ILogicalOperator dataSource = forwardOp.getInputs().get(0).getValue(); builder.contributeGraphEdge(dataSource, 0, forwardOp, 0); ILogicalOperator rangemapSource = forwardOp.getInputs().get(1).getValue(); builder.contributeGraphEdge(rangemapSource, 0, forwardOp, 1); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { EmptyTupleSourceRuntimeFactory runtime = new EmptyTupleSourceRuntimeFactory(); runtime.setSourceLocation(op.getSourceLocation()); RecordDescriptor recDesc = new RecordDescriptor(new ISerializerDeserializer[] {}); builder.contributeMicroOperator(op, runtime, recDesc); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { List<Mutable<ILogicalOperator>> inputs = op.getInputs(); for (int i = 0; i < inputs.size(); i++) { ILogicalOperator src = inputs.get(i).getValue(); builder.contributeGraphEdge(src, 0, op, i); } } }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); UnionAllOperatorDescriptor opDesc = new UnionAllOperatorDescriptor(builder.getJobSpec(), op.getInputs().size(), recordDescriptor); opDesc.setSourceLocation(op.getSourceLocation()); contributeOpDesc(builder, (AbstractLogicalOperator) op, opDesc); super.contributeRuntimeOperator(builder, context, op, opSchema, inputSchemas, outerPlanSchema); } }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); MaterializingOperatorDescriptor materializationOpDesc = new MaterializingOperatorDescriptor(builder.getJobSpec(), recDescriptor, isSingleActivity); materializationOpDesc.setSourceLocation(op.getSourceLocation()); contributeOpDesc(builder, (AbstractLogicalOperator) op, materializationOpDesc); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); }
IMetadataProvider mp = context.getMetadataProvider(); JobSpecification spec = builder.getJobSpec(); IOperatorDescriptor opDesc = runtimeAndConstraints.first; opDesc.setSourceLocation(resultOp.getSourceLocation()); builder.contributeHyracksOperator(resultOp, opDesc); builder.contributeGraphEdge(src, 0, resultOp, 0);
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { RecordDescriptor recordDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); MicroUnionAllRuntimeFactory runtime = new MicroUnionAllRuntimeFactory(op.getInputs().size()); runtime.setSourceLocation(op.getSourceLocation()); builder.contributeMicroOperator(op, runtime, recordDescriptor); super.contributeRuntimeOperator(builder, context, op, opSchema, inputSchemas, outerPlanSchema); } }
private void reviseEdges(IHyracksJobBuilder builder) { /* * revise the edges for the case of replicate operator */ operatorVisitedToParents.forEach((child, parents) -> { if (parents.size() > 1) { if (child.getValue().getOperatorTag() == LogicalOperatorTag.REPLICATE || child.getValue().getOperatorTag() == LogicalOperatorTag.SPLIT) { AbstractReplicateOperator rop = (AbstractReplicateOperator) child.getValue(); if (rop.isBlocker()) { // make the order of the graph edges consistent with the order of rop's outputs List<Mutable<ILogicalOperator>> outputs = rop.getOutputs(); for (Mutable<ILogicalOperator> parent : parents) { builder.contributeGraphEdge(child.getValue(), outputs.indexOf(parent), parent.getValue(), 0); } } else { int i = 0; for (Mutable<ILogicalOperator> parent : parents) { builder.contributeGraphEdge(child.getValue(), i, parent.getValue(), 0); i++; } } } } }); } }
@SuppressWarnings({ "rawtypes", "unchecked" }) @Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { InsertDeleteUpsertOperator insertDeleteOp = (InsertDeleteUpsertOperator) op; assert insertDeleteOp.getOperation() == Kind.INSERT; assert insertDeleteOp.isBulkload(); IMetadataProvider mp = context.getMetadataProvider(); IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op); JobSpecification spec = builder.getJobSpec(); RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor( context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context); Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getInsertRuntime(dataSource, propagatedSchema, typeEnv, primaryKeys, payload, additionalFilteringKeys, additionalNonFilterVars, inputDesc, context, spec, true); IOperatorDescriptor opDesc = runtimeAndConstraints.first; opDesc.setSourceLocation(insertDeleteOp.getSourceLocation()); builder.contributeHyracksOperator(insertDeleteOp, opDesc); builder.contributeAlgebricksPartitionConstraint(opDesc, runtimeAndConstraints.second); ILogicalOperator src = insertDeleteOp.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, insertDeleteOp, 0); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { ProjectOperator project = (ProjectOperator) op; int[] projectionList = new int[project.getVariables().size()]; int i = 0; for (LogicalVariable v : project.getVariables()) { int pos = inputSchemas[0].findVariable(v); if (pos < 0) { throw new AlgebricksException("Could not find variable " + v + "."); } projectionList[i++] = pos; } StreamProjectRuntimeFactory runtime = new StreamProjectRuntimeFactory(projectionList, flushFramesRapidly); runtime.setSourceLocation(project.getSourceLocation()); RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); builder.contributeMicroOperator(project, runtime, recDesc); ILogicalOperator src = project.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, project, 0); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { IOperatorDescriptorRegistry spec = builder.getJobSpec(); RecordDescriptor recDescriptor = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); ReplicateOperator rop = (ReplicateOperator) op; int outputArity = rop.getOutputArity(); boolean[] outputMaterializationFlags = rop.getOutputMaterializationFlags(); ReplicateOperatorDescriptor ropDesc = new ReplicateOperatorDescriptor(spec, recDescriptor, outputArity, outputMaterializationFlags); ropDesc.setSourceLocation(rop.getSourceLocation()); contributeOpDesc(builder, rop, ropDesc); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { Pair<IConnectorDescriptor, TargetConstraint> connPair = createConnectorDescriptor(builder.getJobSpec(), op, opSchema, context); builder.contributeConnectorWithTargetConstraint(op, connPair.first, connPair.second); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { propagatedSchema.addAllVariables(outerPlanSchema); NestedTupleSourceRuntimeFactory runtime = new NestedTupleSourceRuntimeFactory(); runtime.setSourceLocation(op.getSourceLocation()); RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), propagatedSchema, context); builder.contributeMicroOperator(op, runtime, recDesc); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { IndexInsertDeleteUpsertOperator indexInsertDeleteOp = (IndexInsertDeleteUpsertOperator) op; assert indexInsertDeleteOp.getOperation() == Kind.INSERT; assert indexInsertDeleteOp.isBulkload(); IMetadataProvider mp = context.getMetadataProvider(); IVariableTypeEnvironment typeEnv = context.getTypeEnvironment(op); JobSpecification spec = builder.getJobSpec(); RecordDescriptor inputDesc = JobGenHelper.mkRecordDescriptor( context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context); Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> runtimeAndConstraints = mp.getIndexInsertRuntime(dataSourceIndex, propagatedSchema, inputSchemas, typeEnv, primaryKeys, secondaryKeys, additionalFilteringKeys, filterExpr, inputDesc, context, spec, true); IOperatorDescriptor opDesc = runtimeAndConstraints.first; opDesc.setSourceLocation(indexInsertDeleteOp.getSourceLocation()); builder.contributeHyracksOperator(indexInsertDeleteOp, opDesc); builder.contributeAlgebricksPartitionConstraint(opDesc, runtimeAndConstraints.second); ILogicalOperator src = indexInsertDeleteOp.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, indexInsertDeleteOp, 0); }
@Override public void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, ILogicalOperator op, IOperatorSchema opSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { SubplanOperator subplan = (SubplanOperator) op; if (subplan.getNestedPlans().size() != 1) { throw new NotImplementedException("Subplan currently works only for one nested plan with one root."); } List<List<AlgebricksPipeline>> subplans = compileSubplansImpl(inputSchemas[0], subplan, opSchema, context); assert subplans.size() == 1; List<AlgebricksPipeline> np = subplans.get(0); RecordDescriptor inputRecordDesc = JobGenHelper.mkRecordDescriptor( context.getTypeEnvironment(op.getInputs().get(0).getValue()), inputSchemas[0], context); IMissingWriterFactory[] missingWriterFactories = new IMissingWriterFactory[np.get(0).getOutputWidth()]; for (int i = 0; i < missingWriterFactories.length; i++) { missingWriterFactories[i] = context.getMissingWriterFactory(); } RecordDescriptor recDesc = JobGenHelper.mkRecordDescriptor(context.getTypeEnvironment(op), opSchema, context); SubplanRuntimeFactory runtime = new SubplanRuntimeFactory(np, missingWriterFactories, inputRecordDesc, recDesc, null); runtime.setSourceLocation(subplan.getSourceLocation()); builder.contributeMicroOperator(subplan, runtime, recDesc); ILogicalOperator src = op.getInputs().get(0).getValue(); builder.contributeGraphEdge(src, 0, op, 0); }