public static boolean setOperatorMode(AbstractLogicalOperator op) { AbstractLogicalOperator.ExecutionMode oldMode = op.getExecutionMode(); switch (op.getOperatorTag()) { case DATASOURCESCAN: { op.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED); AbstractLogicalOperator currentOp = op; while (currentOp.getInputs().size() == 1) { AbstractLogicalOperator child = (AbstractLogicalOperator) currentOp.getInputs().get(0).getValue(); if (child.getOperatorTag() != LogicalOperatorTag.EMPTYTUPLESOURCE) { break; child.setExecutionMode(AbstractLogicalOperator.ExecutionMode.PARTITIONED); currentOp = child; AbstractLogicalOperator prevOp = (AbstractLogicalOperator) nts.getDataSourceReference().getValue().getInputs().get(0).getValue(); if (prevOp.getExecutionMode() != AbstractLogicalOperator.ExecutionMode.UNPARTITIONED) { nts.setExecutionMode(AbstractLogicalOperator.ExecutionMode.LOCAL); if (op.getOperatorTag() == LogicalOperatorTag.LIMIT) { LimitOperator opLim = (LimitOperator) op; if (opLim.isTopmostLimitOp()) { if (op.getOperatorTag() == LogicalOperatorTag.AGGREGATE) { AggregateOperator aggOp = (AggregateOperator) op; if (aggOp.isGlobal()) { op.setExecutionMode(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED); forceUnpartitioned = true;
public static void computeSchemaAndPropertiesRecIfNull(AbstractLogicalOperator op, IOptimizationContext context) throws AlgebricksException { if (op.getSchema() == null) { for (Mutable<ILogicalOperator> i : op.getInputs()) { computeSchemaAndPropertiesRecIfNull((AbstractLogicalOperator) i.getValue(), context); } if (op.hasNestedPlans()) { AbstractOperatorWithNestedPlans a = (AbstractOperatorWithNestedPlans) op; for (ILogicalPlan p : a.getNestedPlans()) { for (Mutable<ILogicalOperator> r : p.getRoots()) { computeSchemaAndPropertiesRecIfNull((AbstractLogicalOperator) r.getValue(), context); } } } op.recomputeSchema(); op.computeDeliveredPhysicalProperties(context); } }
public static ILogicalOperator deepCopy(ILogicalOperator op) throws AlgebricksException { OperatorDeepCopyVisitor visitor = new OperatorDeepCopyVisitor(); AbstractLogicalOperator copiedOperator = (AbstractLogicalOperator) op.accept(visitor, null); copiedOperator.setSourceLocation(op.getSourceLocation()); copiedOperator.setExecutionMode(op.getExecutionMode()); copiedOperator.getAnnotations().putAll(op.getAnnotations()); copiedOperator.setSchema(op.getSchema()); AbstractLogicalOperator sourceOp = (AbstractLogicalOperator) op; copiedOperator.setPhysicalOperator(sourceOp.getPhysicalOperator()); return copiedOperator; }
public static Mutable<ILogicalOperator> findLastSubplanOperator(Mutable<ILogicalOperator> opRef) { AbstractLogicalOperator op = (AbstractLogicalOperator) opRef; AbstractLogicalOperator next; while (op.getOperatorTag() != LogicalOperatorTag.NESTEDTUPLESOURCE) { opRef = op.getInputs().get(0); op = (AbstractLogicalOperator) opRef; next = (AbstractLogicalOperator) op.getInputs().get(0).getValue(); if (next.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) { break; } } return opRef; }
private boolean tryingToPushThroughSelectionWithSameDataSource(AssignOperator access, AbstractLogicalOperator op2) { if (op2.getOperatorTag() != LogicalOperatorTag.SELECT) { return false; } ILogicalExpression e1 = (ILogicalExpression) access.getAnnotations().get(OperatorAnnotation.FIELD_ACCESS); if (e1 == null) { return false; } ILogicalExpression e2 = (ILogicalExpression) op2.getAnnotations().get(OperatorAnnotation.FIELD_ACCESS); if (e2 == null) { return false; } return e1.equals(e2); }
private void disableJobGenRec(ILogicalOperator operator) { AbstractLogicalOperator op = (AbstractLogicalOperator) operator; op.disableJobGen(); for (Mutable<ILogicalOperator> i : op.getInputs()) { disableJobGenRec(i.getValue()); } }
PhysicalRequirements pr = op.getRequiredPhysicalPropertiesForChildren(required, context); IPhysicalPropertiesVector[] reqdProperties = null; if (pr != null) { for (Mutable<ILogicalOperator> childRef : op.getInputs()) { AbstractLogicalOperator child = (AbstractLogicalOperator) childRef.getValue(); changed |= physOptimizeOp(childRef, reqdProperties[j], nestedPlan, context); child.computeDeliveredPhysicalProperties(context); IPhysicalPropertiesVector delivered = child.getDeliveredPhysicalProperties(); INodeDomain childDomain = delivered.getPartitioningProperty().getNodeDomain(); if (childrenDomain == null) { for (j = 0; j < op.getInputs().size(); j++) { int childIndex = (j + startChildIndex) % op.getInputs().size(); IPhysicalPropertiesVector requiredProperty = reqdProperties[childIndex]; AbstractLogicalOperator child = (AbstractLogicalOperator) op.getInputs().get(childIndex).getValue(); IPhysicalPropertiesVector delivered = child.getDeliveredPhysicalProperties(); .trace(">>>> Properties delivered by " + child.getPhysicalOperator() + ": " + delivered + "\n"); .trace(">>>> Required properties for " + child.getPhysicalOperator() + ": " + rqd + "\n"); AbstractLogicalOperator newChild = (AbstractLogicalOperator) op.getInputs().get(childIndex).getValue(); delivered = newChild.getDeliveredPhysicalProperties(); IPhysicalPropertiesVector newDiff = newPropertiesDiff(newChild, rqd, mayExpandPartitioningProperties, context);
rop.setExecutionMode(((AbstractLogicalOperator) candidate.getValue()).getExecutionMode()); if (aopCandidate.getOperatorTag() == LogicalOperatorTag.EXCHANGE) { rop.getInputs().add(candidate); } else { AbstractLogicalOperator beforeExchange = new ExchangeOperator(); beforeExchange.setPhysicalOperator(new OneToOneExchangePOperator()); beforeExchange.setExecutionMode(rop.getExecutionMode()); Mutable<ILogicalOperator> beforeExchangeRef = new MutableObject<ILogicalOperator>(beforeExchange); beforeExchange.getInputs().add(candidate); context.computeAndSetTypeEnvironmentForOperator(beforeExchange); rop.getInputs().add(beforeExchangeRef); int index = parent.getInputs().indexOf(candidate); if (parent.getOperatorTag() == LogicalOperatorTag.EXCHANGE) { parent.getInputs().set(index, ropRef); rop.getOutputs().add(parentRef); } else { AbstractLogicalOperator exchange = new ExchangeOperator(); exchange.setPhysicalOperator(new OneToOneExchangePOperator()); exchange.setExecutionMode(rop.getExecutionMode()); MutableObject<ILogicalOperator> exchangeRef = new MutableObject<ILogicalOperator>(exchange); exchange.getInputs().add(ropRef); rop.getOutputs().add(exchangeRef); context.computeAndSetTypeEnvironmentForOperator(exchange); parent.getInputs().set(index, exchangeRef); context.computeAndSetTypeEnvironmentForOperator(parent); assignOperator.setSourceLocation(refSourceLoc); assignOperator.setExecutionMode(rop.getExecutionMode());
private static void computeDefaultPhysicalOp(AbstractLogicalOperator op, boolean topLevelOp, IOptimizationContext context) throws AlgebricksException { PhysicalOptimizationConfig physicalOptimizationConfig = context.getPhysicalOptimizationConfig(); if (op.getOperatorTag().equals(LogicalOperatorTag.GROUP)) { GroupByOperator gby = (GroupByOperator) op; if (gby.getNestedPlans().size() == 1) { op.setPhysicalOperator(externalGby); setToExternalGby = true; op.setPhysicalOperator(new PreclusteredGroupByPOperator(columnList, gby.isGroupAll(), context.getPhysicalOptimizationConfig().getMaxFramesForGroupBy())); op.setPhysicalOperator(new PreclusteredGroupByPOperator(columnList, gby.isGroupAll(), context.getPhysicalOptimizationConfig().getMaxFramesForGroupBy())); } else { if (op.getPhysicalOperator() == null) { switch (op.getOperatorTag()) { case INNERJOIN: { JoinUtils.setJoinAlgorithmAndExchangeAlgo((InnerJoinOperator) op, topLevelOp, context); INodeDomain storageDomain = mp.findNodeDomain(dataset.getNodeGroupName()); if (dsi == null) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, op.getSourceLocation(), "Could not find index " + jobGenParams.getIndexName() + " for dataset " + dataSourceId); BTreeJobGenParams btreeJobGenParams = new BTreeJobGenParams(); btreeJobGenParams.readFromFuncArgs(f.getArguments());
private void setNewOp(Mutable<ILogicalOperator> opRef, AbstractLogicalOperator newOp, IOptimizationContext context) throws AlgebricksException { ILogicalOperator oldOp = opRef.getValue(); opRef.setValue(newOp); newOp.getInputs().add(new MutableObject<>(oldOp)); newOp.recomputeSchema(); newOp.computeDeliveredPhysicalProperties(context); context.computeAndSetTypeEnvironmentForOperator(newOp); if (AlgebricksConfig.ALGEBRICKS_LOGGER.isTraceEnabled()) { AlgebricksConfig.ALGEBRICKS_LOGGER.trace(">>>> Structural properties for " + newOp.getPhysicalOperator() + ": " + newOp.getDeliveredPhysicalProperties() + "\n"); } PhysicalOptimizationsUtil.computeFDsAndEquivalenceClasses(newOp, context); }
@Override public void printOperator(AbstractLogicalOperator op, int indent) throws AlgebricksException { final AlgebricksAppendable out = this.get(); op.accept(this, indent); IPhysicalOperator pOp = op.getPhysicalOperator(); if (pOp != null) { out.append("\n"); pad(out, indent); appendln(out, "-- " + pOp.toString() + " |" + op.getExecutionMode() + "|"); } else { appendln(out, " -- |" + op.getExecutionMode() + "|"); } for (Mutable<ILogicalOperator> i : op.getInputs()) { printOperator((AbstractLogicalOperator) i.getValue(), indent + 2); } }
public static void printPhysicalOperator(AbstractLogicalOperator op, int indent, AlgebricksAppendable out) throws AlgebricksException { IPhysicalOperator pOp = op.getPhysicalOperator(); pad(out, indent); appendln(out, "-- " + pOp.toString() + " |" + op.getExecutionMode() + "|"); if (op.hasNestedPlans()) { AbstractOperatorWithNestedPlans opNest = (AbstractOperatorWithNestedPlans) op; for (ILogicalPlan p : opNest.getNestedPlans()) { pad(out, indent + 8); appendln(out, "{"); printPhysicalOps(p, out, indent + 10); pad(out, indent + 8); appendln(out, "}"); } } for (Mutable<ILogicalOperator> i : op.getInputs()) { printPhysicalOperator((AbstractLogicalOperator) i.getValue(), indent + 2, out); } }
switch (op.getOperatorTag()) { case AGGREGATE: case SUBPLAN: switch (op.getOperatorTag()) { case UNNEST: case DATASOURCESCAN: { if (op.hasNestedPlans() && op.getOperatorTag() != LogicalOperatorTag.SUBPLAN) { AbstractOperatorWithNestedPlans opWithNestedPlans = (AbstractOperatorWithNestedPlans) op; opWithNestedPlans.getUsedVariablesExceptNestedPlans(targetUsedVars); if (!op.hasInputs()) { if (!belowSecondUnnest) { return findPlanPartition((AbstractLogicalOperator) op.getInputs().get(0).getValue(), innerUsedVars, outerUsedVars, innerOps, outerOps, topSelects, belowSecondUnnest);
/** * Adds the free variables of the plan rooted at that operator to the * collection provided. * * @param op * @param freeVars * - The collection to which the free variables will be added. */ public static void getFreeVariablesInSelfOrDesc(AbstractLogicalOperator op, Set<LogicalVariable> freeVars) throws AlgebricksException { HashSet<LogicalVariable> produced = new HashSet<>(); VariableUtilities.getProducedVariables(op, produced); for (LogicalVariable v : produced) { freeVars.remove(v); } HashSet<LogicalVariable> used = new HashSet<>(); VariableUtilities.getUsedVariables(op, used); for (LogicalVariable v : used) { freeVars.add(v); } if (op.hasNestedPlans()) { AbstractOperatorWithNestedPlans s = (AbstractOperatorWithNestedPlans) op; getFreeVariablesInSubplans(s, freeVars); } for (Mutable<ILogicalOperator> i : op.getInputs()) { getFreeVariablesInSelfOrDesc((AbstractLogicalOperator) i.getValue(), freeVars); } }
switch (op1.getOperatorTag()) { case SINK: case DELEGATE_OPERATOR: { if (op1.getOperatorTag() == LogicalOperatorTag.DELEGATE_OPERATOR) { DelegateOperator eOp = (DelegateOperator) op1; if (!(eOp.getDelegate() instanceof CommitOperator)) { AbstractLogicalOperator op2 = (AbstractLogicalOperator) op1.getInputs().get(0).getValue(); if (op2.getOperatorTag() == LogicalOperatorTag.INSERT_DELETE_UPSERT) { InsertDeleteUpsertOperator insertDeleteOp = (InsertDeleteUpsertOperator) op2; if (insertDeleteOp.getOperation() == InsertDeleteUpsertOperator.Kind.DELETE) { requiredRecordType = (ARecordType) op1.getAnnotations().get("output-record-type"); if (requiredRecordType == null) { return false; IVariableTypeEnvironment env = op.computeOutputTypeEnvironment(context); IAType inputRecordType = (IAType) env.getVarType(recordVar); boolean cast = !compatible(requiredRecordType, inputRecordType, op.getSourceLocation());
if (inputOp.getOperatorTag() == LogicalOperatorTag.PROJECT || context.checkAndAddToAlreadyCompared(assignOp, inputOp) && !(inputOp.getOperatorTag() == LogicalOperatorTag.SELECT && isAccessToIndexedField(assignOp, context))) { return false; Object annotation = inputOp.getAnnotations().get(OperatorPropertiesUtil.MOVABLE); if (annotation != null && !((Boolean) annotation)) { return false; if (inputOp.getOperatorTag() == LogicalOperatorTag.GROUP) { VariableUtilities.getLiveVariables(inputOp, produced2); } else { if (inter.isEmpty()) { pushItDown = true; } else if (inputOp.getOperatorTag() == LogicalOperatorTag.GROUP) { GroupByOperator g = (GroupByOperator) inputOp; List<Pair<LogicalVariable, LogicalVariable>> varMappings = new ArrayList<>(); if (inputOp.getOperatorTag() == LogicalOperatorTag.NESTEDTUPLESOURCE) { Mutable<ILogicalOperator> childOfSubplan = ((NestedTupleSourceOperator) inputOp).getDataSourceReference().getValue().getInputs().get(0); if (inputOp.getInputs().size() == 1 && !inputOp.hasNestedPlans()) { pushAccessDown(opRef, inputOp, inputOp.getInputs().get(0), context, finalAnnot); return true; } else { for (Mutable<ILogicalOperator> inp : inputOp.getInputs()) {
@Override public void computeDeliveredProperties(ILogicalOperator iop, IOptimizationContext context) throws AlgebricksException { IPartitioningProperty pp; AbstractLogicalOperator op = (AbstractLogicalOperator) iop; if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED) { AbstractLogicalOperator op0 = (AbstractLogicalOperator) op.getInputs().get(0).getValue(); IPhysicalPropertiesVector pv0 = op0.getPhysicalOperator().getDeliveredProperties(); AbstractLogicalOperator op1 = (AbstractLogicalOperator) op.getInputs().get(1).getValue(); IPhysicalPropertiesVector pv1 = op1.getPhysicalOperator().getDeliveredProperties(); if (pv0 == null || pv1 == null) { pp = null; } else { pp = pv0.getPartitioningProperty(); } } else { pp = IPartitioningProperty.UNPARTITIONED; } this.deliveredProperties = new StructuralPropertiesVector(pp, deliveredLocalProperties(iop, context)); }
throws AlgebricksException { AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); if (!op.hasInputs()) { return false; for (int i = 0; i < op.getInputs().size(); i++) { AbstractLogicalOperator childOp = (AbstractLogicalOperator) op.getInputs().get(i).getValue(); if (childOp.getOperatorTag() != LogicalOperatorTag.ASSIGN) { continue; if (childOfChildOp.getOperatorTag() != LogicalOperatorTag.UNIONALL) { continue; op.getInputs().set(i, assignOp.getInputs().get(0)); context.computeAndSetTypeEnvironmentForOperator(op); modified = true;
do { if (subTreeOp.getOperatorTag() == LogicalOperatorTag.SELECT) { searchOpRef = subTreeOp.getInputs().get(0); subTreeOp = (AbstractLogicalOperator) searchOpRef.getValue(); if (subTreeOp.getOperatorTag() != LogicalOperatorTag.ASSIGN && subTreeOp.getOperatorTag() != LogicalOperatorTag.UNNEST) { if (!subTreeOp.getInputs().isEmpty()) { searchOpRef = subTreeOp.getInputs().get(0); subTreeOp = (AbstractLogicalOperator) searchOpRef.getValue(); while (subTreeOp.getOperatorTag() == LogicalOperatorTag.ASSIGN || subTreeOp.getOperatorTag() == LogicalOperatorTag.UNNEST) { if (!passedSource && !OperatorPropertiesUtil.isMovable(subTreeOp)) { return false; if (subTreeOp.getExecutionMode() != ExecutionMode.UNPARTITIONED) { searchOpRef = subTreeOp.getInputs().get(0); subTreeOp = (AbstractLogicalOperator) searchOpRef.getValue(); return result; } while (subTreeOp.getOperatorTag() == LogicalOperatorTag.SELECT);
private boolean testIfExchangeAbove(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); if (op.getOperatorTag() == LogicalOperatorTag.EXCHANGE) { return false; } boolean exchInserted = false; for (Mutable<ILogicalOperator> i : op.getInputs()) { AbstractLogicalOperator c = (AbstractLogicalOperator) i.getValue(); IPhysicalOperator cpop = c.getPhysicalOperator(); if (c.getOperatorTag() == LogicalOperatorTag.EXCHANGE || cpop == null) { continue; } if (!cpop.isMicroOperator()) { insertOneToOneExchange(i, context); exchInserted = true; } } return exchInserted; }