private boolean isInvertedIndexSearch(AbstractUnnestMapOperator op) { IPhysicalOperator physicalOperator = op.getPhysicalOperator(); final PhysicalOperatorTag physicalOperatorTag = physicalOperator.getOperatorTag(); if (physicalOperatorTag == PhysicalOperatorTag.LENGTH_PARTITIONED_INVERTED_INDEX_SEARCH || physicalOperatorTag == PhysicalOperatorTag.SINGLE_PARTITION_INVERTED_INDEX_SEARCH) { return true; } return false; }
@Override public final void computeDeliveredPhysicalProperties(IOptimizationContext context) throws AlgebricksException { physicalOperator.computeDeliveredProperties(this, context); }
@Override public final void contributeRuntimeOperator(IHyracksJobBuilder builder, JobGenContext context, IOperatorSchema propagatedSchema, IOperatorSchema[] inputSchemas, IOperatorSchema outerPlanSchema) throws AlgebricksException { if (bJobGenEnabled) { if (physicalOperator == null) { throw AlgebricksException.create(PHYS_OPERATOR_NOT_SET, getSourceLocation(), this.getOperatorTag()); } physicalOperator.contributeRuntimeOperator(builder, context, this, propagatedSchema, inputSchemas, outerPlanSchema); } }
if (!(op.getPhysicalOperator().getOperatorTag() == aop.getPhysicalOperator().getOperatorTag())) { return Boolean.FALSE; IPhysicalPropertiesVector properties = op.getPhysicalOperator().getDeliveredProperties(); IPhysicalPropertiesVector propertiesArg = aop.getPhysicalOperator().getDeliveredProperties(); if (properties == null && propertiesArg == null) { return Boolean.TRUE;
private boolean testIfExchangeBelow(Mutable<ILogicalOperator> opRef, IOptimizationContext context) throws AlgebricksException { AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); boolean exchInserted = false; for (Mutable<ILogicalOperator> i : op.getInputs()) { AbstractLogicalOperator c = (AbstractLogicalOperator) i.getValue(); if (c.getOperatorTag() != LogicalOperatorTag.EXCHANGE) { if (c.getPhysicalOperator() == null) { return false; } insertOneToOneExchange(i, context); exchInserted = true; } } IPhysicalOperator pt = op.getPhysicalOperator(); if (pt.isJobGenDisabledBelowMe() || arrayContains(operatorsBelowWhichJobGenIsDisabled, pt.getOperatorTag())) { for (Mutable<ILogicalOperator> i : op.getInputs()) { disableJobGenRec(i.getValue()); } } return exchInserted; }
/** * @return the physical properties that this operator delivers, based on * what its children deliver */ @Override public final IPhysicalPropertiesVector getDeliveredPhysicalProperties() { return physicalOperator.getDeliveredProperties(); }
protected boolean worthMaterialization(Mutable<ILogicalOperator> candidate) { AbstractLogicalOperator aop = (AbstractLogicalOperator) candidate.getValue(); if (aop.getPhysicalOperator().expensiveThanMaterialization()) { return true; } List<Mutable<ILogicalOperator>> inputs = candidate.getValue().getInputs(); for (Mutable<ILogicalOperator> inputRef : inputs) { if (worthMaterialization(inputRef)) { return true; } } return false; } }
Pair<int[], int[]> labels = aop.getPhysicalOperator().getInputOutputDependencyLabels(opRef.getValue()); List<Mutable<ILogicalOperator>> inputs = opRef.getValue().getInputs(); for (int i = 0; i < inputs.size(); i++) {
@Override protected List<ILocalStructuralProperty> deliveredLocalProperties(ILogicalOperator op, IOptimizationContext context) { AbstractLogicalOperator op0 = (AbstractLogicalOperator) op.getInputs().get(0).getValue(); IPhysicalPropertiesVector pv0 = op0.getPhysicalOperator().getDeliveredProperties(); List<ILocalStructuralProperty> lp0 = pv0.getLocalProperties(); if (lp0 != null) { // maintains the local properties on the probe side return new LinkedList<ILocalStructuralProperty>(lp0); } return new LinkedList<ILocalStructuralProperty>(); }
private boolean isBlockingGroupBy(GroupByOperator op) { return op.getPhysicalOperator().getOperatorTag() == PhysicalOperatorTag.EXTERNAL_GROUP_BY || op.getPhysicalOperator().getOperatorTag() == PhysicalOperatorTag.SORT_GROUP_BY; }
stringBuilder.append(", Exec: ").append(op.getExecutionMode()); if (show) { IPhysicalPropertiesVector properties = physicalOp == null ? null : physicalOp.getDeliveredProperties(); List<ILocalStructuralProperty> localProp = properties == null ? null : properties.getLocalProperties(); IPartitioningProperty partitioningProp = properties == null ? null : properties.getPartitioningProperty();
private long getExchangeRequiredMemory(ExchangeOperator op) { final IPhysicalOperator physicalOperator = op.getPhysicalOperator(); final PhysicalOperatorTag physicalOperatorTag = physicalOperator.getOperatorTag(); if (physicalOperatorTag == PhysicalOperatorTag.ONE_TO_ONE_EXCHANGE || physicalOperatorTag == PhysicalOperatorTag.SORT_MERGE_EXCHANGE) { return getOperatorRequiredMemory(op, frameSize); } return 2L * MAX_BUFFER_PER_CONNECTION * numComputationPartitions * numComputationPartitions * frameSize; }
@Override public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) { AbstractLogicalOperator op2; List<ILocalStructuralProperty> propsLocal = new ArrayList<ILocalStructuralProperty>(); IPhysicalPropertiesVector childsProperties = null; for (int i = 0; i < op.getInputs().size(); i++) { op2 = (AbstractLogicalOperator) op.getInputs().get(i).getValue(); childsProperties = op2.getPhysicalOperator().getDeliveredProperties(); if (childsProperties.getLocalProperties() != null) { propsLocal.addAll(childsProperties.getLocalProperties()); } } deliveredProperties = new StructuralPropertiesVector(childsProperties.getPartitioningProperty(), propsLocal); }
private void calculateMemoryUsageForExchange(ExchangeOperator op) throws AlgebricksException { visitInternal(op, false); IPhysicalOperator physicalOperator = op.getPhysicalOperator(); PhysicalOperatorTag physicalOperatorTag = physicalOperator.getOperatorTag(); if (physicalOperatorTag == PhysicalOperatorTag.ONE_TO_ONE_EXCHANGE || physicalOperatorTag == PhysicalOperatorTag.SORT_MERGE_EXCHANGE) { addOutputBuffer(op); return; } stageMemorySoFar += 2L * MAX_BUFFER_PER_CONNECTION * numComputationPartitions * numComputationPartitions * frameSize; clusterCapacity.setAggregatedMemoryByteSize(stageMemorySoFar); }
@Override public void computeDeliveredProperties(ILogicalOperator op, IOptimizationContext context) { AbstractLogicalOperator op2 = (AbstractLogicalOperator) op.getInputs().get(0).getValue(); IPhysicalPropertiesVector childsProperties = op2.getPhysicalOperator().getDeliveredProperties(); List<ILocalStructuralProperty> propsLocal = new ArrayList<>(); if (childsProperties.getLocalProperties() != null) { propsLocal.addAll(childsProperties.getLocalProperties()); } // ... get local properties for newly created variables... SubplanOperator subplan = (SubplanOperator) op; for (ILogicalPlan plan : subplan.getNestedPlans()) { for (Mutable<ILogicalOperator> r : plan.getRoots()) { AbstractLogicalOperator rOp = (AbstractLogicalOperator) r.getValue(); propsLocal.addAll(rOp.getPhysicalOperator().getDeliveredProperties().getLocalProperties()); } } deliveredProperties = new StructuralPropertiesVector(childsProperties.getPartitioningProperty(), propsLocal); }
private void validateOperatorStages(List<PlanStage> stages, ILogicalOperator operator) { if (stages.size() == 1) { return; } long expectedAppearances = BLOCKING_OPERATORS.contains(operator.getOperatorTag()) ? 2 : 1; if (operator.getOperatorTag() == GROUP) { GroupByOperator groupByOperator = (GroupByOperator) operator; if (groupByOperator.getPhysicalOperator().getOperatorTag() == PhysicalOperatorTag.EXTERNAL_GROUP_BY || groupByOperator.getPhysicalOperator().getOperatorTag() == PhysicalOperatorTag.SORT_GROUP_BY) { expectedAppearances = 2; } } final long actual = stages.stream().map(PlanStage::getOperators).filter(op -> op.contains(operator)).count(); Assert.assertEquals(expectedAppearances, actual); }
@Override public void computeDeliveredProperties(ILogicalOperator iop, IOptimizationContext context) throws AlgebricksException { IPartitioningProperty pp; AbstractLogicalOperator op = (AbstractLogicalOperator) iop; if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED) { AbstractLogicalOperator op0 = (AbstractLogicalOperator) op.getInputs().get(0).getValue(); IPhysicalPropertiesVector pv0 = op0.getPhysicalOperator().getDeliveredProperties(); AbstractLogicalOperator op1 = (AbstractLogicalOperator) op.getInputs().get(1).getValue(); IPhysicalPropertiesVector pv1 = op1.getPhysicalOperator().getDeliveredProperties(); if (pv0 == null || pv1 == null) { pp = null; } else { pp = pv0.getPartitioningProperty(); } } else { pp = IPartitioningProperty.UNPARTITIONED; } this.deliveredProperties = new StructuralPropertiesVector(pp, deliveredLocalProperties(iop, context)); }
private boolean isRedundantSort(Mutable<ILogicalOperator> opRef, IPhysicalPropertiesVector delivered, IPhysicalPropertiesVector diffOfProperties, IOptimizationContext context) { AbstractLogicalOperator op = (AbstractLogicalOperator) opRef.getValue(); if (op.getOperatorTag() != LogicalOperatorTag.ORDER || (op.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.STABLE_SORT && op.getPhysicalOperator().getOperatorTag() != PhysicalOperatorTag.IN_MEMORY_STABLE_SORT) || delivered.getLocalProperties() == null) { return false; } AbstractStableSortPOperator sortOp = (AbstractStableSortPOperator) op.getPhysicalOperator(); sortOp.computeLocalProperties(op); ILocalStructuralProperty orderProp = sortOp.getOrderProperty(); return PropertiesUtil.matchLocalProperties(Collections.singletonList(orderProp), delivered.getLocalProperties(), context.getEquivalenceClassMap(op), context.getFDList(op)); }
IPhysicalPropertiesVector probeSideProperties = probeOp.getPhysicalOperator().getDeliveredProperties(); List<ILocalStructuralProperty> probeSideLocalProperties = probeSideProperties.getLocalProperties(); if (probeSideLocalProperties != null) {
/** * When true, the sort operator requires ORDERED_PARTITION (only applicable to dynamic version for now). * Conditions: * 1. Execution mode == partitioned * 2. Dynamic range map was not disabled by some checks * 3. User didn't disable it * 4. User didn't provide static range map * 5. Physical sort operator is not in-memory * 6. There are at least two partitions in the cluster * @param sortOp the sort operator * @param clusterDomain the partitions specification of the cluster * @param ctx optimization context * @return true if the sort operator should be full parallel sort, false otherwise. */ private boolean isFullParallel(AbstractLogicalOperator sortOp, INodeDomain clusterDomain, IOptimizationContext ctx) { return sortOp.getAnnotations().get(OperatorAnnotations.USE_DYNAMIC_RANGE) != Boolean.FALSE && !sortOp.getAnnotations().containsKey(OperatorAnnotations.USE_STATIC_RANGE) && sortOp.getPhysicalOperator().getOperatorTag() == PhysicalOperatorTag.STABLE_SORT && clusterDomain.cardinality() != null && clusterDomain.cardinality() > 1 && ctx.getPhysicalOptimizationConfig().getSortParallel(); } }