public RequiredCapacityVisitor(int numComputationPartitions, int sortFrameLimit, int groupFrameLimit, int joinFrameLimit, int frameSize, IClusterCapacity clusterCapacity) { this.numComputationPartitions = numComputationPartitions; this.frameSize = frameSize; this.groupByMemorySize = groupFrameLimit * (long) frameSize; this.joinMemorySize = joinFrameLimit * (long) frameSize; this.sortMemorySize = sortFrameLimit * (long) frameSize; this.clusterCapacity = clusterCapacity; this.clusterCapacity.setAggregatedCores(1); // At least one core is needed. }
private NodeCapacity getAdjustedNodeCapacity(NodeCapacity nodeCapacity) { return new NodeCapacity(nodeCapacity.getMemoryByteSize(), nodeCapacity.getCores() * nodeCoresMultiplier); }
@Test public void test() throws HyracksException { ClusterCapacity capacity = new ClusterCapacity(); String nodeId = "node1"; capacity.update(nodeId, new NodeCapacity(1024L, 8)); Assert.assertTrue(capacity.getAggregatedMemoryByteSize() == 1024L); Assert.assertTrue(capacity.getAggregatedCores() == 8); capacity.update(nodeId, new NodeCapacity(-1L, -2)); Assert.assertTrue(capacity.getAggregatedMemoryByteSize() == 0L); Assert.assertTrue(capacity.getAggregatedCores() == 0); capacity.getMemoryByteSize(nodeId); } catch (HyracksException e) { nodeNotExist = e.getErrorCode() == ErrorCode.NO_SUCH_NODE; nodeNotExist = false; try { capacity.getCores(nodeId); } catch (HyracksException e) { nodeNotExist = e.getErrorCode() == ErrorCode.NO_SUCH_NODE; capacity.update(nodeId, new NodeCapacity(1024L, 8)); capacity.update(nodeId, new NodeCapacity(4L, 0)); Assert.assertTrue(capacity.getAggregatedMemoryByteSize() == 0L); Assert.assertTrue(capacity.getAggregatedCores() == 0);
@Override public JobSubmissionStatus allocate(JobSpecification job) throws HyracksException { IClusterCapacity requiredCapacity = job.getRequiredClusterCapacity(); long reqAggregatedMemoryByteSize = requiredCapacity.getAggregatedMemoryByteSize(); int reqAggregatedNumCores = requiredCapacity.getAggregatedCores(); IReadOnlyClusterCapacity maximumCapacity = resourceManager.getMaximumCapacity(); if (!(reqAggregatedMemoryByteSize <= maximumCapacity.getAggregatedMemoryByteSize() && reqAggregatedNumCores <= maximumCapacity.getAggregatedCores())) { throw HyracksException.create(ErrorCode.JOB_REQUIREMENTS_EXCEED_CAPACITY, requiredCapacity.toString(), maximumCapacity.toString()); } IClusterCapacity currentCapacity = resourceManager.getCurrentCapacity(); long currentAggregatedMemoryByteSize = currentCapacity.getAggregatedMemoryByteSize(); int currentAggregatedAvailableCores = currentCapacity.getAggregatedCores(); if (!(reqAggregatedMemoryByteSize <= currentAggregatedMemoryByteSize && reqAggregatedNumCores <= currentAggregatedAvailableCores)) { return JobSubmissionStatus.QUEUE; } currentCapacity.setAggregatedMemoryByteSize(currentAggregatedMemoryByteSize - reqAggregatedMemoryByteSize); currentCapacity.setAggregatedCores(currentAggregatedAvailableCores - reqAggregatedNumCores); return JobSubmissionStatus.EXECUTE; }
@Override public NodeCapacity getCapacity() { return new NodeCapacity(Runtime.getRuntime().maxMemory(), Runtime.getRuntime().availableProcessors() - 1); }
@Override public void release(JobSpecification job) { IClusterCapacity requiredCapacity = job.getRequiredClusterCapacity(); long reqAggregatedMemoryByteSize = requiredCapacity.getAggregatedMemoryByteSize(); int reqAggregatedNumCores = requiredCapacity.getAggregatedCores(); IClusterCapacity currentCapacity = resourceManager.getCurrentCapacity(); long aggregatedMemoryByteSize = currentCapacity.getAggregatedMemoryByteSize(); int aggregatedNumCores = currentCapacity.getAggregatedCores(); currentCapacity.setAggregatedMemoryByteSize(aggregatedMemoryByteSize + reqAggregatedMemoryByteSize); currentCapacity.setAggregatedCores(aggregatedNumCores + reqAggregatedNumCores); ensureMaxCapacity(); }
public static IClusterCapacity getStageBasedRequiredCapacity(List<PlanStage> stages, int computationLocations, int sortFrameLimit, int groupFrameLimit, int joinFrameLimit, int textSearchFrameLimit, int frameSize) { final OperatorResourcesComputer computer = new OperatorResourcesComputer(computationLocations, sortFrameLimit, groupFrameLimit, joinFrameLimit, textSearchFrameLimit, frameSize); final IClusterCapacity clusterCapacity = new ClusterCapacity(); final Long maxRequiredMemory = stages.stream().mapToLong(stage -> stage.getRequiredMemory(computer)).max() .orElseThrow(IllegalStateException::new); clusterCapacity.setAggregatedMemoryByteSize(maxRequiredMemory); final Integer maxRequireCores = stages.stream().mapToInt(stage -> stage.getRequiredCores(computer)).max() .orElseThrow(IllegalStateException::new); clusterCapacity.setAggregatedCores(maxRequireCores); return clusterCapacity; } }
private void ensureMaxCapacity() { final IClusterCapacity currentCapacity = resourceManager.getCurrentCapacity(); final IReadOnlyClusterCapacity maximumCapacity = resourceManager.getMaximumCapacity(); if (currentCapacity.getAggregatedCores() > maximumCapacity.getAggregatedCores() || currentCapacity.getAggregatedMemoryByteSize() > maximumCapacity.getAggregatedMemoryByteSize()) { LOGGER.warn("Current cluster available capacity {} is more than its maximum capacity {}", currentCapacity, maximumCapacity); } } }
@Override public void update(String nodeId, NodeCapacity nodeCapacity) throws HyracksException { // Removes the existing node resource and the aggregated resource statistics. if (nodeMemoryMap.containsKey(nodeId)) { aggregatedMemoryByteSize -= nodeMemoryMap.remove(nodeId); } if (nodeCoreMap.containsKey(nodeId)) { aggregatedCores -= nodeCoreMap.remove(nodeId); } long memorySize = nodeCapacity.getMemoryByteSize(); int cores = nodeCapacity.getCores(); // Updates the node capacity map when both memory size and cores are positive. if (memorySize > 0 && cores > 0) { aggregatedMemoryByteSize += memorySize; aggregatedCores += cores; nodeMemoryMap.put(nodeId, memorySize); nodeCoreMap.put(nodeId, cores); } }
private void addOutputBuffer(ILogicalOperator op) { if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED || op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.LOCAL) { stageMemorySoFar += frameSize * numComputationPartitions; // every operator needs one output buffer. } else { stageMemorySoFar += frameSize; // every operator needs one output buffer. } clusterCapacity.setAggregatedMemoryByteSize(stageMemorySoFar); }
@Override public Map<String, NodeControllerInfo> getNodeControllerInfoMap() { Map<String, NodeControllerInfo> result = new LinkedHashMap<>(); nodeRegistry.forEach( (key, ncState) -> result.put(key, new NodeControllerInfo(key, NodeStatus.ACTIVE, ncState.getDataPort(), ncState.getResultPort(), ncState.getMessagingPort(), ncState.getCapacity().getCores()))); return result; }
private void assertRequiredMemory(List<PlanStage> stages, long expectedMemory) { final IClusterCapacity clusterCapacity = ResourceUtils.getStageBasedRequiredCapacity(stages, PARALLELISM, FRAME_LIMIT, FRAME_LIMIT, FRAME_LIMIT, FRAME_LIMIT, FRAME_SIZE); Assert.assertEquals(clusterCapacity.getAggregatedMemoryByteSize(), expectedMemory); } }
private void releaseJobCapacity(JobRun jobRun) { final JobSpecification job = jobRun.getJobSpecification(); jobCapacityController.release(job); } }
@Override public NodeCapacity getCapacity() { int allCores = ManagementFactory.getOperatingSystemMXBean().getAvailableProcessors(); return new NodeCapacity(Runtime.getRuntime().maxMemory(), allCores > 1 ? allCores - 1 : allCores); }
private void calculateMemoryUsageForBlockingOperators(ILogicalOperator op, long memSize) throws AlgebricksException { visitInternal(op, false); if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED || op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.LOCAL) { stageMemorySoFar += memSize * numComputationPartitions; } else { stageMemorySoFar += memSize; } clusterCapacity.setAggregatedMemoryByteSize(stageMemorySoFar); }
private void setAvailableCores(ILogicalOperator op) { if (op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.PARTITIONED || op.getExecutionMode() == AbstractLogicalOperator.ExecutionMode.LOCAL) { clusterCapacity.setAggregatedCores((int) numComputationPartitions); } } }
@Override public NodeCapacity getCapacity() { StorageProperties storageProperties = runtimeContext.getStorageProperties(); final long memorySize = storageProperties.getJobExecutionMemoryBudget(); int allCores = Runtime.getRuntime().availableProcessors(); return new NodeCapacity(memorySize, allCores); }
private void calculateMemoryUsageForExchange(ExchangeOperator op) throws AlgebricksException { visitInternal(op, false); IPhysicalOperator physicalOperator = op.getPhysicalOperator(); PhysicalOperatorTag physicalOperatorTag = physicalOperator.getOperatorTag(); if (physicalOperatorTag == PhysicalOperatorTag.ONE_TO_ONE_EXCHANGE || physicalOperatorTag == PhysicalOperatorTag.SORT_MERGE_EXCHANGE) { addOutputBuffer(op); return; } stageMemorySoFar += 2L * MAX_BUFFER_PER_CONNECTION * numComputationPartitions * numComputationPartitions * frameSize; clusterCapacity.setAggregatedMemoryByteSize(stageMemorySoFar); }
@Override @Idempotent public synchronized void removeNode(String nodeId) throws HyracksException { NodeControllerState ncState = nodeRegistry.remove(nodeId); if (ncState == null) { LOGGER.warn("request to remove unknown node {}; ignoring", nodeId); } else { removeNodeFromIpAddressMap(nodeId, ncState); } // Updates the cluster capacity (idempotent) resourceManager.update(nodeId, new NodeCapacity(0L, 0)); }