public RequiredCapacityVisitor(int numComputationPartitions, int sortFrameLimit, int groupFrameLimit, int joinFrameLimit, int frameSize, IClusterCapacity clusterCapacity) { this.numComputationPartitions = numComputationPartitions; this.frameSize = frameSize; this.groupByMemorySize = groupFrameLimit * (long) frameSize; this.joinMemorySize = joinFrameLimit * (long) frameSize; this.sortMemorySize = sortFrameLimit * (long) frameSize; this.clusterCapacity = clusterCapacity; this.clusterCapacity.setAggregatedCores(1); // At least one core is needed. }
private void setAvailableCores(ILogicalOperator op) { if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED || op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.LOCAL) { clusterCapacity.setAggregatedCores((int) numComputationPartitions); } } }
public static IClusterCapacity getStageBasedRequiredCapacity(List<PlanStage> stages, int computationLocations, int sortFrameLimit, int groupFrameLimit, int joinFrameLimit, int textSearchFrameLimit, int frameSize) { final OperatorResourcesComputer computer = new OperatorResourcesComputer(computationLocations, sortFrameLimit, groupFrameLimit, joinFrameLimit, textSearchFrameLimit, frameSize); final IClusterCapacity clusterCapacity = new ClusterCapacity(); final Long maxRequiredMemory = stages.stream().mapToLong(stage -> stage.getRequiredMemory(computer)).max() .orElseThrow(IllegalStateException::new); clusterCapacity.setAggregatedMemoryByteSize(maxRequiredMemory); final Integer maxRequireCores = stages.stream().mapToInt(stage -> stage.getRequiredCores(computer)).max() .orElseThrow(IllegalStateException::new); clusterCapacity.setAggregatedCores(maxRequireCores); return clusterCapacity; } }
@Override public JobSubmissionStatus allocate(JobSpecification job) throws HyracksException { IClusterCapacity requiredCapacity = job.getRequiredClusterCapacity(); long reqAggregatedMemoryByteSize = requiredCapacity.getAggregatedMemoryByteSize(); int reqAggregatedNumCores = requiredCapacity.getAggregatedCores(); IReadOnlyClusterCapacity maximumCapacity = resourceManager.getMaximumCapacity(); if (!(reqAggregatedMemoryByteSize <= maximumCapacity.getAggregatedMemoryByteSize() && reqAggregatedNumCores <= maximumCapacity.getAggregatedCores())) { throw HyracksException.create(ErrorCode.JOB_REQUIREMENTS_EXCEED_CAPACITY, requiredCapacity.toString(), maximumCapacity.toString()); } IClusterCapacity currentCapacity = resourceManager.getCurrentCapacity(); long currentAggregatedMemoryByteSize = currentCapacity.getAggregatedMemoryByteSize(); int currentAggregatedAvailableCores = currentCapacity.getAggregatedCores(); if (!(reqAggregatedMemoryByteSize <= currentAggregatedMemoryByteSize && reqAggregatedNumCores <= currentAggregatedAvailableCores)) { return JobSubmissionStatus.QUEUE; } currentCapacity.setAggregatedMemoryByteSize(currentAggregatedMemoryByteSize - reqAggregatedMemoryByteSize); currentCapacity.setAggregatedCores(currentAggregatedAvailableCores - reqAggregatedNumCores); return JobSubmissionStatus.EXECUTE; }
@Override public void release(JobSpecification job) { IClusterCapacity requiredCapacity = job.getRequiredClusterCapacity(); long reqAggregatedMemoryByteSize = requiredCapacity.getAggregatedMemoryByteSize(); int reqAggregatedNumCores = requiredCapacity.getAggregatedCores(); IClusterCapacity currentCapacity = resourceManager.getCurrentCapacity(); long aggregatedMemoryByteSize = currentCapacity.getAggregatedMemoryByteSize(); int aggregatedNumCores = currentCapacity.getAggregatedCores(); currentCapacity.setAggregatedMemoryByteSize(aggregatedMemoryByteSize + reqAggregatedMemoryByteSize); currentCapacity.setAggregatedCores(aggregatedNumCores + reqAggregatedNumCores); ensureMaxCapacity(); }