@Override public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { ILogicalOperator op = opRef.getValue(); if (context.checkIfInDontApplySet(this, op)) { return false; } op.accept(visitor, null); context.computeAndSetTypeEnvironmentForOperator(op); context.addToDontApplySet(this, op); return false; }
@Override protected FunctionDataSource toDatasource(IOptimizationContext context, AbstractFunctionCallExpression f) throws AlgebricksException { return new PingDatasource(context.getComputationNodeDomain()); }
protected void setMetadataDeclarations(IOptimizationContext context) { metadataProvider = (MetadataProvider) context.getMetadataProvider(); }
private Map<LogicalVariable, EquivalenceClass> getOrCreateEqClasses(ILogicalOperator op, IOptimizationContext ctx) throws AlgebricksException { Map<LogicalVariable, EquivalenceClass> eqClasses = ctx.getEquivalenceClassMap(op); if (eqClasses == null) { eqClasses = new HashMap<LogicalVariable, EquivalenceClass>(); ctx.putEquivalenceClassMap(op, eqClasses); } return eqClasses; }
@Override public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { ILogicalOperator op = opRef.getValue(); if (context.getOutputTypeEnvironment(op) != null) { return false; } context.computeAndSetTypeEnvironmentForOperator(op); return true; }
Set<LogicalVariable> noKeyVars = new HashSet<>(); for (LogicalVariable liveVar : liveVars) { List<LogicalVariable> keyVars = context.findPrimaryKey(liveVar); if (keyVars != null) { keyVars.retainAll(liveVars); return new Pair<ILogicalOperator, Set<LogicalVariable>>(operator, primaryKeyVars); } else { LogicalVariable assignVar = context.newVar(); ILogicalOperator assignOp = new AssignOperator(assignVar, new MutableObject<ILogicalExpression>(new StatefulFunctionCallExpression( OperatorPropertiesUtil.markMovable(assignOp, !usedForCorrelationJoin); assignOp.getInputs().add(new MutableObject<ILogicalOperator>(operator)); context.addPrimaryKey(new FunctionalDependency(Collections.singletonList(assignVar), new ArrayList<LogicalVariable>(liveVars))); context.computeAndSetTypeEnvironmentForOperator(assignOp); return new Pair<ILogicalOperator, Set<LogicalVariable>>(assignOp, Collections.singleton(assignVar));
return false; if (context.checkIfInDontApplySet(this, op)) { return false; List<FunctionalDependency> fds = context.getFDList(op); if (fds != null && !fds.isEmpty()) { return false; physicalOptimizationConfig = context.getPhysicalOptimizationConfig(); if (AlgebricksConfig.ALGEBRICKS_LOGGER.isTraceEnabled()) { AlgebricksConfig.ALGEBRICKS_LOGGER.trace(">>>> Optimizing operator " + op.getPhysicalOperator() + ".\n"); new StructuralPropertiesVector(new RandomPartitioningProperty(context.getComputationNodeDomain()), new LinkedList<ILocalStructuralProperty>()); boolean changed = physOptimizeOp(opRef, pvector, false, context); context.addToDontApplySet(this, opRef.getValue());
private void propagateFilterInSecondaryUnnsetMap(UnnestMapOperator secondaryUnnest, IAType filterType, IOptimizationContext context) throws AlgebricksException { LogicalVariable minIndexFilterVar = context.newVar(); LogicalVariable maxIndexFilterVar = context.newVar(); secondaryUnnest.markPropagageIndexFilter(); secondaryUnnest.getVariables().add(minIndexFilterVar); secondaryUnnest.getVariableTypes().add(filterType); secondaryUnnest.getVariables().add(maxIndexFilterVar); secondaryUnnest.getVariableTypes().add(filterType); context.computeAndSetTypeEnvironmentForOperator(secondaryUnnest); }
@Override public ILogicalOperator visitDistinctOperator(DistinctOperator op, Void arg) throws AlgebricksException { visitSingleInputOperator(op); List<LogicalVariable> distinctVarList = op.getDistinctByVarList(); for (LogicalVariable keyVar : correlatedKeyVars) { if (!distinctVarList.contains(keyVar)) { distinctVarList.add(keyVar); } } context.computeAndSetTypeEnvironmentForOperator(op); return op; }
@Override public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); // current operator is INNERJOIN or LEFTOUTERJOIN or SELECT Mutable<ILogicalExpression> expRef; if (op.getOperatorTag() == LogicalOperatorTag.INNERJOIN || op.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN) { AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) op; expRef = joinOp.getCondition(); } else if (op.getOperatorTag() == LogicalOperatorTag.SELECT) { SelectOperator selectOp = (SelectOperator) op; expRef = selectOp.getCondition(); } else { return false; } MetadataProvider metadataProvider = ((MetadataProvider) context.getMetadataProvider()); IVariableTypeEnvironment env = context.getOutputTypeEnvironment(op); if (expandFuzzyEq(expRef, context, env, metadataProvider)) { context.computeAndSetTypeEnvironmentForOperator(op); return true; } return false; }
protected static LogicalVariable extractExprIntoAssignOpRef(ILogicalExpression gExpr, Mutable<ILogicalOperator> opRef2, IOptimizationContext context) throws AlgebricksException { LogicalVariable v = context.newVar(); AssignOperator a = new AssignOperator(v, new MutableObject<>(gExpr)); a.setSourceLocation(gExpr.getSourceLocation()); a.getInputs().add(new MutableObject<>(opRef2.getValue())); opRef2.setValue(a); if (gExpr.getExpressionTag() == LogicalExpressionTag.CONSTANT) { context.addNotToBeInlinedVar(v); } context.computeAndSetTypeEnvironmentForOperator(a); return v; }
IFunctionInfo samplingFun = context.getMetadataProvider().lookupFunction(localSamplingFun); AbstractFunctionCallExpression samplingExp = new AggregateFunctionCallExpression(samplingFun, false, sampledFields); samplingExp.setSourceLocation(sourceLocation); LogicalVariable samplingResultVar = context.newVar(); localResultVariables.add(samplingResultVar); localAggFunctions.add(new MutableObject<>(samplingExp)); Object[] samplingParam = { context.getPhysicalOptimizationConfig().getSortSamples() }; samplingExp.setOpaqueParameters(samplingParam); IFunctionInfo rangeMapFun = context.getMetadataProvider().lookupFunction(rangeMapFunction); AbstractFunctionCallExpression rangeMapExp = new AggregateFunctionCallExpression(rangeMapFun, true, arg); rangeMapExp.setSourceLocation(sourceLocation); globalResultVariables.add(context.newVar()); globalAggFunctions.add(new MutableObject<>(rangeMapExp));
@Override public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); if (context.checkIfInDontApplySet(this, op)) { return false; } computeDefaultPhysicalOp(op, true, context); context.addToDontApplySet(this, op); return true; }
/** * Generate new ORDER operator that uses TopKSort module and replaces the old ORDER operator. */ private boolean pushLimitIntoOrder(Mutable<ILogicalOperator> opRef, Mutable<ILogicalOperator> opRef2, IOptimizationContext context) throws AlgebricksException { PhysicalOptimizationConfig physicalOptimizationConfig = context.getPhysicalOptimizationConfig(); LimitOperator limitOp = (LimitOperator) opRef.getValue(); OrderOperator orderOp = (OrderOperator) opRef2.getValue(); // We don't push-down LIMIT into in-memory sort. if (orderOp.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.STABLE_SORT) { return false; } Integer topK = getOutputLimit(limitOp); if (topK == null) { return false; } // Create the new ORDER operator, set the topK value, and replace the current one. OrderOperator newOrderOp = new OrderOperator(orderOp.getOrderExpressions(), topK); newOrderOp.setSourceLocation(orderOp.getSourceLocation()); newOrderOp.setPhysicalOperator( new StableSortPOperator(physicalOptimizationConfig.getMaxFramesExternalSort(), newOrderOp.getTopK())); newOrderOp.getInputs().addAll(orderOp.getInputs()); newOrderOp.setExecutionMode(orderOp.getExecutionMode()); newOrderOp.recomputeSchema(); newOrderOp.computeDeliveredPhysicalProperties(context); opRef2.setValue(newOrderOp); context.computeAndSetTypeEnvironmentForOperator(newOrderOp); context.addToDontApplySet(this, limitOp); return true; }
private void addKeyVarsAndExprs(IOptimizableFuncExpr optFuncExpr, ArrayList<LogicalVariable> keyVarList, ArrayList<Mutable<ILogicalExpression>> keyExprList, IOptimizationContext context) throws AlgebricksException { // For now we are assuming a single secondary index key. // Add a variable and its expr to the lists which will be passed into an assign op. LogicalVariable keyVar = context.newVar(); keyVarList.add(keyVar); keyExprList.add(new MutableObject<ILogicalExpression>(optFuncExpr.getConstantExpr(0))); return; }
@Override public StorageComponentsDatasource toDatasource(IOptimizationContext context, AbstractFunctionCallExpression f) throws AlgebricksException { String dataverseName = getString(f.getArguments(), 0); String datasetName = getString(f.getArguments(), 1); MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider(); Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName); if (dataset == null) { throw new CompilationException(ErrorCode.UNKNOWN_DATASET_IN_DATAVERSE, f.getSourceLocation(), datasetName, dataverseName); } return new StorageComponentsDatasource(context.getComputationNodeDomain(), dataset.getDatasetId()); } }
private IPhysicalOperator createHashConnector(IOptimizationContext ctx, IPhysicalPropertiesVector deliveredByChild, INodeDomain domain, IPhysicalPropertiesVector requiredAtChild, IPartitioningProperty rqdPartitioning, int childIndex, ILogicalOperator parentOp) { IPhysicalOperator hashConnector; List<LogicalVariable> vars = new ArrayList<>(((UnorderedPartitionedProperty) rqdPartitioning).getColumnSet()); String hashMergeHint = (String) ctx.getMetadataProvider().getConfig().get(HASH_MERGE); if (hashMergeHint == null || !hashMergeHint.equalsIgnoreCase(TRUE_CONSTANT)) { hashConnector = new HashPartitionExchangePOperator(vars, domain); return hashConnector; } List<ILocalStructuralProperty> cldLocals = deliveredByChild.getLocalProperties(); List<ILocalStructuralProperty> reqdLocals = requiredAtChild.getLocalProperties(); boolean propWasSet = false; hashConnector = null; if (reqdLocals != null && cldLocals != null && allAreOrderProps(cldLocals)) { AbstractLogicalOperator c = (AbstractLogicalOperator) parentOp.getInputs().get(childIndex).getValue(); Map<LogicalVariable, EquivalenceClass> ecs = ctx.getEquivalenceClassMap(c); List<FunctionalDependency> fds = ctx.getFDList(c); if (PropertiesUtil.matchLocalProperties(reqdLocals, cldLocals, ecs, fds)) { List<OrderColumn> orderColumns = getOrderColumnsFromGroupingProperties(reqdLocals, cldLocals); hashConnector = new HashPartitionMergeExchangePOperator(orderColumns, vars, domain); propWasSet = true; } } if (!propWasSet) { hashConnector = new HashPartitionExchangePOperator(vars, domain); } return hashConnector; }
@Override public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { if (context.checkIfInDontApplySet(this, opRef.getValue())) { return false; } boolean modified = removeRedundantVariables(opRef, context); if (modified) { context.computeAndSetTypeEnvironmentForOperator(opRef.getValue()); } return modified; }
childrenDomain = delivered.getPartitioningProperty().getNodeDomain(); } else if (!childrenDomain.sameAs(childDomain)) { childrenDomain = context.getComputationNodeDomain(); mayExpandPartitioningProperties, context.getEquivalenceClassMap(child), context.getFDList(child));
private void typeOpRec(Mutable<ILogicalOperator> r, IOptimizationContext context) throws AlgebricksException { AbstractLogicalOperator op = (AbstractLogicalOperator) r.getValue(); for (Mutable<ILogicalOperator> i : op.getInputs()) { typeOpRec(i, context); } if (op.hasNestedPlans()) { for (ILogicalPlan p : ((AbstractOperatorWithNestedPlans) op).getNestedPlans()) { typePlan(p, context); } } context.computeAndSetTypeEnvironmentForOperator(op); context.addToDontApplySet(this, op); }