/** * After the pattern is matched, checks the condition and initializes the data source * from the right (inner) sub tree. * * @throws AlgebricksException */ protected boolean checkJoinOpConditionAndInitSubTree(IOptimizationContext context) throws AlgebricksException { typeEnvironment = context.getOutputTypeEnvironment(joinOp); // Check that the join's condition is a function call. ILogicalExpression condExpr = joinOp.getCondition().getValue(); if (condExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) { return false; } joinCond = (AbstractFunctionCallExpression) condExpr; // The result of the left subtree initialization does not need to be checked since only the field type // of the field that is being joined is important. However, if we do not initialize the left sub tree, // we lose a chance to get the field type of a field if there is an enforced index on it. leftSubTree.initFromSubTree(joinOp.getInputs().get(0)); boolean rightSubTreeInitialized = rightSubTree.initFromSubTree(joinOp.getInputs().get(1)); if (!rightSubTreeInitialized) { return false; } // The right (inner) subtree must have a datasource scan. if (rightSubTree.hasDataSourceScan()) { return true; } return false; }
public static void setJoinAlgorithmAndExchangeAlgo(AbstractBinaryJoinOperator op, boolean topLevelOp, IOptimizationContext context) throws AlgebricksException { if (!topLevelOp) { throw new IllegalStateException("Micro operator not implemented for: " + op.getOperatorTag()); List<LogicalVariable> varsLeft = op.getInputs().get(0).getValue().getSchema(); List<LogicalVariable> varsRight = op.getInputs().get(1).getValue().getSchema(); if (isHashJoinCondition(op.getCondition().getValue(), varsLeft, varsRight, sideLeft, sideRight)) { BroadcastSide side = getBroadcastJoinSide(op.getCondition().getValue(), varsLeft, varsRight); if (side == null) { setHashJoinOp(op, JoinPartitioningType.PAIRWISE, sideLeft, sideRight, context); break; case LEFT: if (op.getJoinKind() == AbstractBinaryJoinOperator.JoinKind.INNER) { Mutable<ILogicalOperator> opRef0 = op.getInputs().get(0); Mutable<ILogicalOperator> opRef1 = op.getInputs().get(1); ILogicalOperator tmp = opRef0.getValue(); opRef0.setValue(opRef1.getValue());
/** * Inject variables to indicate non-matches for the right branch of a left-outer join. * * @param joinOp * the leftouter join operator. */ private void injectNullCheckVars(AbstractBinaryJoinOperator joinOp) { LogicalVariable assignVar = context.newVar(); AssignOperator assignOp = new AssignOperator(assignVar, new MutableObject<ILogicalExpression>(ConstantExpression.TRUE)); assignOp.setSourceLocation(joinOp.getSourceLocation()); assignOp.getInputs().add(joinOp.getInputs().get(1)); joinOp.getInputs().set(1, new MutableObject<ILogicalOperator>(assignOp)); nullCheckVars.add(assignVar); }
private static void setNestedLoopJoinOp(AbstractBinaryJoinOperator op, IOptimizationContext context) { op.setPhysicalOperator(new NestedLoopJoinPOperator(op.getJoinKind(), JoinPartitioningType.BROADCAST, context.getPhysicalOptimizationConfig().getMaxFramesForJoin())); }
public static List<Mutable<ILogicalExpression>> getExpressions(Mutable<ILogicalOperator> opRef) { AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); List<Mutable<ILogicalExpression>> result = new ArrayList<Mutable<ILogicalExpression>>(); switch (op.getOperatorTag()) { case AGGREGATE: case ASSIGN: case RUNNINGAGGREGATE: AbstractAssignOperator aao = (AbstractAssignOperator) op; result.addAll(aao.getExpressions()); break; case INNERJOIN: case LEFTOUTERJOIN: AbstractBinaryJoinOperator abjo = (AbstractBinaryJoinOperator) op; result.add(abjo.getCondition()); break; case SELECT: SelectOperator so = (SelectOperator) op; result.add(so.getCondition()); break; case UNNEST: case UNNEST_MAP: AbstractUnnestOperator auo = (AbstractUnnestOperator) op; result.add(auo.getExpressionRef()); break; default: // TODO Not yet implemented. break; } return result; }
Mutable<ILogicalExpression> exprRef = joinOp.getCondition(); Mutable<ILogicalExpression> getItemExprRef = getSimilarityExpression(exprRef); if (getItemExprRef == null) { List<Mutable<ILogicalOperator>> inputOps = joinOp.getInputs(); ILogicalOperator leftInputOp = inputOps.get(0).getValue(); ILogicalOperator rightInputOp = inputOps.get(1).getValue(); switch (joinOp.getJoinKind()) { case INNER: { extraSelect = new SelectOperator(exprRef, false, null);
conditionInputSchemas[0] = propagatedSchema; IExpressionRuntimeProvider expressionRuntimeProvider = context.getExpressionRuntimeProvider(); IScalarEvaluatorFactory cond = expressionRuntimeProvider.createEvaluatorFactory(join.getCondition().getValue(), context.getTypeEnvironment(op), conditionInputSchemas, context); ITuplePairComparatorFactory comparatorFactory = opDesc.setSourceLocation(join.getSourceLocation()); contributeOpDesc(builder, join, opDesc);
private static void hybridToInMemHashJoin(AbstractBinaryJoinOperator op, IOptimizationContext context) throws AlgebricksException { ILogicalOperator opBuild = op.getInputs().get(1).getValue(); LogicalPropertiesVisitor.computeLogicalPropertiesDFS(opBuild, context); ILogicalPropertiesVector v = context.getLogicalPropertiesVector(opBuild); boolean loggerTraceEnabled = AlgebricksConfig.ALGEBRICKS_LOGGER.isTraceEnabled(); if (loggerTraceEnabled) { AlgebricksConfig.ALGEBRICKS_LOGGER.trace("// HybridHashJoin inner branch -- Logical properties for " + opBuild.getOperatorTag() + ": " + v + "\n"); } if (v != null) { int size2 = v.getMaxOutputFrames(); HybridHashJoinPOperator hhj = (HybridHashJoinPOperator) op.getPhysicalOperator(); if (size2 > 0 && size2 * hhj.getFudgeFactor() <= hhj.getMemSizeInFrames()) { if (loggerTraceEnabled) { AlgebricksConfig.ALGEBRICKS_LOGGER .trace("// HybridHashJoin inner branch " + opBuild.getOperatorTag() + " fits in memory\n"); } // maintains the local properties on the probe side op.setPhysicalOperator( new InMemoryHashJoinPOperator(hhj.getKind(), hhj.getPartitioningType(), hhj.getKeysLeftBranch(), hhj.getKeysRightBranch(), v.getNumberOfTuples() * 2, hhj.getMemSizeInFrames())); } } }
private boolean removeUnusedJoin(Mutable<ILogicalOperator> opRef) throws AlgebricksException { AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); boolean modified = false; usedVars.clear(); VariableUtilities.getUsedVariables(op, usedVars); // Propagate used variables from parents downwards. parentsUsedVars.addAll(usedVars); int numInputs = op.getInputs().size(); for (int i = 0; i < numInputs; i++) { Mutable<ILogicalOperator> childOpRef = op.getInputs().get(i); int unusedJoinBranchIndex = removeJoinFromInputBranch(childOpRef); if (unusedJoinBranchIndex >= 0) { int usedBranchIndex = (unusedJoinBranchIndex == 0) ? 1 : 0; // Remove join at input index i, by hooking up op's input i with // the join's branch at unusedJoinBranchIndex. AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) childOpRef.getValue(); op.getInputs().set(i, joinOp.getInputs().get(usedBranchIndex)); modified = true; } // Descend into children. if (removeUnusedJoin(childOpRef)) { modified = true; } } return modified; }
private boolean pushDownFunctions(AbstractBinaryJoinOperator joinOp, int inputIndex, List<Mutable<ILogicalExpression>> funcExprs, IOptimizationContext context) throws AlgebricksException { ILogicalOperator joinInputOp = joinOp.getInputs().get(inputIndex).getValue(); liveVars.clear(); VariableUtilities.getLiveVariables(joinInputOp, liveVars); AssignOperator newAssign = new AssignOperator(assignVars, assignExprs); newAssign.getInputs().add(new MutableObject<ILogicalOperator>(joinInputOp)); newAssign.setExecutionMode(joinOp.getExecutionMode()); joinOp.getInputs().get(inputIndex).setValue(newAssign); context.computeAndSetTypeEnvironmentForOperator(newAssign); return true;
returnOp.setSourceLocation(op.getSourceLocation()); returnOp.getInputs().addAll(op.getInputs()); injectNullCheckVars(returnOp);
switch (joinOp.getJoinKind()) { case INNER: prepareJoin = AQLPLUS_INNER_JOIN + AQLPLUS;
@Override public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); if (op.getOperatorTag() != LogicalOperatorTag.INNERJOIN && op.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) { return false; } AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) op; ILogicalExpression expr = joinOp.getCondition().getValue(); return assignFunctionExpressions(joinOp, expr, context); }
private void push(Mutable<ILogicalOperator> opRefGby, Mutable<ILogicalOperator> opRefJoin, int branch, List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorToPush, List<Pair<LogicalVariable, Mutable<ILogicalExpression>>> decorNotToPush, IOptimizationContext context) throws AlgebricksException { GroupByOperator gby = (GroupByOperator) opRefGby.getValue(); AbstractBinaryJoinOperator join = (AbstractBinaryJoinOperator) opRefJoin.getValue(); gby.getDecorList().clear(); gby.getDecorList().addAll(decorToPush); for (Pair<LogicalVariable, Mutable<ILogicalExpression>> p : decorNotToPush) { LogicalVariable v1 = p.first; if (v1 != null) { VariableReferenceExpression varRef = (VariableReferenceExpression) p.second.getValue(); LogicalVariable v2 = varRef.getVariableReference(); OperatorManipulationUtil.substituteVarRec(join, v2, v1, true, context); } } Mutable<ILogicalOperator> branchRef = join.getInputs().get(branch); ILogicalOperator opBranch = branchRef.getValue(); opRefJoin.setValue(opBranch); branchRef.setValue(gby); opRefGby.setValue(join); }
private static void setHashJoinOp(AbstractBinaryJoinOperator op, JoinPartitioningType partitioningType, List<LogicalVariable> sideLeft, List<LogicalVariable> sideRight, IOptimizationContext context) throws AlgebricksException { op.setPhysicalOperator(new HybridHashJoinPOperator(op.getJoinKind(), partitioningType, sideLeft, sideRight, context.getPhysicalOptimizationConfig().getMaxFramesForJoin(), context.getPhysicalOptimizationConfig().getMaxFramesForJoinLeftInput(), context.getPhysicalOptimizationConfig().getMaxRecordsPerFrame(), context.getPhysicalOptimizationConfig().getFudgeFactor())); if (partitioningType == JoinPartitioningType.BROADCAST) { hybridToInMemHashJoin(op, context); } }
@Override public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { ILogicalOperator op = opRef.getValue(); if (op.getOperatorTag() != LogicalOperatorTag.INNERJOIN && op.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) { return false; } AbstractBinaryJoinOperator joinOperator = (AbstractBinaryJoinOperator) op; ILogicalOperator left = joinOperator.getInputs().get(0).getValue(); ILogicalOperator right = joinOperator.getInputs().get(1).getValue(); if (!joinOperator.getCondition().getValue().equals(ConstantExpression.TRUE)) { return false; } if (emptyBranch(left)) { opRef.setValue(right); return true; } if (emptyBranch(right)) { opRef.setValue(left); return true; } return false; }
@Override public boolean rewritePost(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); if (op.getOperatorTag() != LogicalOperatorTag.INNERJOIN && op.getOperatorTag() != LogicalOperatorTag.LEFTOUTERJOIN) { return false; } AbstractBinaryJoinOperator joinOp = (AbstractBinaryJoinOperator) op; ILogicalExpression expr = joinOp.getCondition().getValue(); return assignFunctionExpressions(joinOp, expr, context); }
/** * Does the actual push of aggregates for qualified joins. * * @param join * @param assignOp * that contains aggregate function calls. * @param context * @throws AlgebricksException */ private boolean pushAggregateFunctionThroughJoin(AbstractBinaryJoinOperator join, AssignOperator assignOp, IOptimizationContext context) throws AlgebricksException { boolean applied = false; for (Mutable<ILogicalOperator> branchRef : join.getInputs()) { AbstractLogicalOperator branch = (AbstractLogicalOperator) branchRef.getValue(); if (branch.getOperatorTag() == LogicalOperatorTag.AGGREGATE) { AggregateOperator aggOp = (AggregateOperator) branch; applied |= pushAggregateFunction(aggOp, assignOp, context); } else if (branch.getOperatorTag() == LogicalOperatorTag.INNERJOIN || branch.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN) { AbstractBinaryJoinOperator childJoin = (AbstractBinaryJoinOperator) branch; applied |= pushAggregateFunctionThroughJoin(childJoin, assignOp, context); } } return applied; }
/** * Check whether the join is aggregate-pushable, that is, * 1) the join condition is true; * 2) each join branch produces only one tuple. * * @param join * @return true if pushable */ private boolean pushableThroughJoin(AbstractBinaryJoinOperator join) { ILogicalExpression condition = join.getCondition().getValue(); if (condition.equals(ConstantExpression.TRUE)) { // Checks if the aggregation functions are pushable through the join boolean pushable = true; for (Mutable<ILogicalOperator> branchRef : join.getInputs()) { AbstractLogicalOperator branch = (AbstractLogicalOperator) branchRef.getValue(); if (branch.getOperatorTag() == LogicalOperatorTag.AGGREGATE) { pushable &= true; } else if (branch.getOperatorTag() == LogicalOperatorTag.INNERJOIN || branch.getOperatorTag() == LogicalOperatorTag.LEFTOUTERJOIN) { AbstractBinaryJoinOperator childJoin = (AbstractBinaryJoinOperator) branch; pushable &= pushableThroughJoin(childJoin); } else { pushable &= false; } } return pushable; } return false; }
private static void addCondToJoin(SelectOperator select, AbstractBinaryJoinOperator join, IOptimizationContext context) { ILogicalExpression cond = join.getCondition().getValue(); if (OperatorPropertiesUtil.isAlwaysTrueCond(cond)) { // the join was a product join.getCondition().setValue(select.getCondition().getValue()); } else { boolean bAddedToConj = false; if (cond.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) { AbstractFunctionCallExpression fcond = (AbstractFunctionCallExpression) cond; if (fcond.getFunctionIdentifier().equals(AlgebricksBuiltinFunctions.AND)) { AbstractFunctionCallExpression newCond = new ScalarFunctionCallExpression( context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.AND)); newCond.getArguments().add(select.getCondition()); newCond.getArguments().addAll(fcond.getArguments()); join.getCondition().setValue(newCond); bAddedToConj = true; } } if (!bAddedToConj) { AbstractFunctionCallExpression newCond = new ScalarFunctionCallExpression( context.getMetadataProvider().lookupFunction(AlgebricksBuiltinFunctions.AND), select.getCondition(), new MutableObject<ILogicalExpression>(join.getCondition().getValue())); join.getCondition().setValue(newCond); } } }