private long getOperatorRequiredMemory(ILogicalOperator op, long memorySize) { if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED || op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.LOCAL) { return memorySize * numComputationPartitions; } return memorySize; }
public int getOperatorRequiredCores(ILogicalOperator operator) { if (operator.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED || operator.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.LOCAL) { return numComputationPartitions; } return MIN_OPERATOR_CORES; }
private void setAvailableCores(ILogicalOperator op) { if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED || op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.LOCAL) { clusterCapacity.setAggregatedCores((int) numComputationPartitions); } } }
private void addOutputBuffer(ILogicalOperator op) { if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED || op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.LOCAL) { stageMemorySoFar += frameSize * numComputationPartitions; // every operator needs one output buffer. } else { stageMemorySoFar += frameSize; // every operator needs one output buffer. } clusterCapacity.setAggregatedMemoryByteSize(stageMemorySoFar); }
private void calculateMemoryUsageForBlockingOperators(ILogicalOperator op, long memSize) throws AlgebricksException { visitInternal(op, false); if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED || op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.LOCAL) { stageMemorySoFar += memSize * numComputationPartitions; } else { stageMemorySoFar += memSize; } clusterCapacity.setAggregatedMemoryByteSize(stageMemorySoFar); }
/** * Checks whether a binary input operator in consideration needs to * run in a single partition mode. If it does, returns an empty properties vector. * Otherwise, returns the proposed partitioned properties vector. * * @param op, the binary input operator in consideration. * @param partitionedPropertiesVector, the proposed partitioned properties vector. * @return either an empty properties vector or the proposed partitioned properties vector. */ public static StructuralPropertiesVector checkUnpartitionedAndGetPropertiesVector(ILogicalOperator op, StructuralPropertiesVector partitionedPropertiesVector) { ILogicalOperator leftChild = op.getInputs().get(0).getValue(); ILogicalOperator rightChild = op.getInputs().get(1).getValue(); boolean unPartitioned = leftChild.getExecutionMode().equals(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED) && rightChild.getExecutionMode().equals(AbstractLogicalOperator.ExecutionMode.UNPARTITIONED); return unPartitioned ? StructuralPropertiesVector.EMPTY_PROPERTIES_VECTOR : partitionedPropertiesVector; } }
private void deepCopyInputsAnnotationsAndExecutionMode(ILogicalOperator op, ILogicalOperator arg, AbstractLogicalOperator opCopy) throws AlgebricksException { deepCopyInputs(op, opCopy, arg); copyAnnotations(op, opCopy); copySourceLocation(op, opCopy); opCopy.setExecutionMode(op.getExecutionMode()); }
private void insertProjectOperator(UnionAllOperator opUnion, int branch, ArrayList<LogicalVariable> usedVariables, IOptimizationContext context) throws AlgebricksException { ProjectOperator projectOp = new ProjectOperator(usedVariables); ILogicalOperator inputOp = opUnion.getInputs().get(branch).getValue(); projectOp.getInputs().add(new MutableObject<ILogicalOperator>(inputOp)); opUnion.getInputs().get(branch).setValue(projectOp); projectOp.setPhysicalOperator(new StreamProjectPOperator()); projectOp.setExecutionMode(inputOp.getExecutionMode()); context.computeAndSetTypeEnvironmentForOperator(projectOp); context.computeAndSetTypeEnvironmentForOperator(inputOp); }
@Override public PhysicalRequirements getRequiredPropertiesForChildren(ILogicalOperator sortOp, IPhysicalPropertiesVector reqdByParent, IOptimizationContext ctx) { if (sortOp.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED) { if (orderProp == null) { computeLocalProperties(sortOp); } StructuralPropertiesVector[] requiredProp = new StructuralPropertiesVector[1]; IPartitioningProperty partitioning; INodeDomain targetNodeDomain = ctx.getComputationNodeDomain(); if (isFullParallel((AbstractLogicalOperator) sortOp, targetNodeDomain, ctx)) { // partitioning requirement: input data is re-partitioned on sort columns (global ordering) // TODO(ali): static range map implementation should be fixed to require ORDERED_PARTITION and come here partitioning = new OrderedPartitionedProperty(Arrays.asList(sortColumns), targetNodeDomain); } else { // partitioning requirement: input data is unpartitioned (i.e. must be merged at one site) partitioning = IPartitioningProperty.UNPARTITIONED; } // local requirement: each partition must be locally ordered requiredProp[0] = new StructuralPropertiesVector(partitioning, Collections.singletonList(orderProp)); return new PhysicalRequirements(requiredProp, IPartitioningRequirementsCoordinator.NO_COORDINATION); } else { return emptyUnaryRequirements(); } }
IPhysicalPropertiesVector reqdByParent, IOptimizationContext context) throws AlgebricksException { IPartitioningProperty pp; switch (op.getExecutionMode()) { case PARTITIONED: pp = new UnorderedPartitionedProperty(new ListSet<>(partitionColumns), break; default: throw new IllegalStateException(op.getExecutionMode().name());
@Override public boolean applySelectPlanTransformation(List<Mutable<ILogicalOperator>> afterSelectRefs, Mutable<ILogicalOperator> selectRef, OptimizableOperatorSubTree subTree, Index chosenIndex, AccessMethodAnalysisContext analysisCtx, IOptimizationContext context) throws AlgebricksException { SelectOperator selectOp = (SelectOperator) selectRef.getValue(); ILogicalOperator indexPlanRootOp = createIndexSearchPlan(afterSelectRefs, selectRef, selectOp.getCondition(), subTree.getAssignsAndUnnestsRefs(), subTree, null, chosenIndex, analysisCtx, false, false, subTree.getDataSourceRef().getValue() .getInputs().get(0).getValue().getExecutionMode() == ExecutionMode.UNPARTITIONED, context, null); // Replace the datasource scan with the new plan rooted at primaryIndexUnnestMap. subTree.getDataSourceRef().setValue(indexPlanRootOp); return true; }
public static ILogicalOperator deepCopy(ILogicalOperator op) throws AlgebricksException { OperatorDeepCopyVisitor visitor = new OperatorDeepCopyVisitor(); AbstractLogicalOperator copiedOperator = (AbstractLogicalOperator) op.accept(visitor, null); copiedOperator.setSourceLocation(op.getSourceLocation()); copiedOperator.setExecutionMode(op.getExecutionMode()); copiedOperator.getAnnotations().putAll(op.getAnnotations()); copiedOperator.setSchema(op.getSchema()); AbstractLogicalOperator sourceOp = (AbstractLogicalOperator) op; copiedOperator.setPhysicalOperator(sourceOp.getPhysicalOperator()); return copiedOperator; }
subTree.getDataSourceRef().getValue(), afterSelectRefs); boolean requiresBroadcast = subTree.getDataSourceRef().getValue().getInputs().get(0).getValue() .getExecutionMode() == ExecutionMode.UNPARTITIONED; ILogicalOperator subRoot = pair.first.createIndexSearchPlan(afterSelectRefs, selectRef, conditionRef, subTree.getAssignsAndUnnestsRefs(), subTree, null, pair.second, analysisCtx, retainInput, false,
StructuralPropertiesVector[] pv = new StructuralPropertiesVector[1]; if (gby.isGroupAll() && gby.isGlobal()) { if (op.getExecutionMode() == ExecutionMode.UNPARTITIONED) { pv[0] = new StructuralPropertiesVector(IPartitioningProperty.UNPARTITIONED, null); return new PhysicalRequirements(pv, IPartitioningRequirementsCoordinator.NO_COORDINATION);
assignOp.setExecutionMode(branchOp.getExecutionMode()); assignOp.getInputs().add(new MutableObject<>(branchOp)); branchOpRef.setValue(assignOp);
exchg.setExecutionMode(childOp.getExecutionMode()); exchg.getInputs().add(new MutableObject<ILogicalOperator>(childOp)); parentOp.getInputs().set(index, new MutableObject<ILogicalOperator>(exchg));
dataset.getDataverseName(), dataset.getDatasetName(), retainInput, scanOperator.getInputs().get(0).getValue() .getExecutionMode() == AbstractLogicalOperator.ExecutionMode.UNPARTITIONED); List<LogicalVariable> empty = new ArrayList<>(); newBTreeParameters.setLowKeyInclusive(true);
new AssignOperator(newAssignVar, new MutableObject<ILogicalExpression>(cast)); newAssignOperator.setSourceLocation(sourceLoc); newAssignOperator.setExecutionMode(op.getExecutionMode()); newAssignOperator.getInputs().add(new MutableObject<ILogicalOperator>(op)); opRef.setValue(newAssignOperator);
limitCloneOp.setExecutionMode(unsafeOp.getExecutionMode()); OperatorPropertiesUtil.computeSchemaRecIfNull((AbstractLogicalOperator) unsafeOp); limitCloneOp.recomputeSchema();
afterSelectRefs), false, subTree.getDataSourceRef().getValue().getInputs().get(0).getValue() .getExecutionMode() == ExecutionMode.UNPARTITIONED, context, null);