@Override public void configure(int maxParallelism) { KeyGroupRangeAssignment.checkParallelismPreconditions(maxParallelism); this.maxParallelism = maxParallelism; } }
@Override public void configure(int maxParallelism) { KeyGroupRangeAssignment.checkParallelismPreconditions(maxParallelism); this.maxParallelism = maxParallelism; } }
@Override public void configure(int maxParallelism) { KeyGroupRangeAssignment.checkParallelismPreconditions(maxParallelism); this.maxParallelism = maxParallelism; } }
@Override public void configure(int maxParallelism) { KeyGroupRangeAssignment.checkParallelismPreconditions(maxParallelism); this.maxParallelism = maxParallelism; } }
/** * Computes a default maximum parallelism from the operator parallelism. This is used in case the user has not * explicitly configured a maximum parallelism to still allow a certain degree of scale-up. * * @param operatorParallelism the operator parallelism as basis for computation. * @return the computed default maximum parallelism. */ public static int computeDefaultMaxParallelism(int operatorParallelism) { checkParallelismPreconditions(operatorParallelism); return Math.min( Math.max( MathUtils.roundUpToPowerOfTwo(operatorParallelism + (operatorParallelism / 2)), DEFAULT_LOWER_BOUND_MAX_PARALLELISM), UPPER_BOUND_MAX_PARALLELISM); }
/** * Computes a default maximum parallelism from the operator parallelism. This is used in case the user has not * explicitly configured a maximum parallelism to still allow a certain degree of scale-up. * * @param operatorParallelism the operator parallelism as basis for computation. * @return the computed default maximum parallelism. */ public static int computeDefaultMaxParallelism(int operatorParallelism) { checkParallelismPreconditions(operatorParallelism); return Math.min( Math.max( MathUtils.roundUpToPowerOfTwo(operatorParallelism + (operatorParallelism / 2)), DEFAULT_LOWER_BOUND_MAX_PARALLELISM), UPPER_BOUND_MAX_PARALLELISM); }
/** * Computes a default maximum parallelism from the operator parallelism. This is used in case the user has not * explicitly configured a maximum parallelism to still allow a certain degree of scale-up. * * @param operatorParallelism the operator parallelism as basis for computation. * @return the computed default maximum parallelism. */ public static int computeDefaultMaxParallelism(int operatorParallelism) { checkParallelismPreconditions(operatorParallelism); return Math.min( Math.max( MathUtils.roundUpToPowerOfTwo(operatorParallelism + (operatorParallelism / 2)), DEFAULT_LOWER_BOUND_MAX_PARALLELISM), UPPER_BOUND_MAX_PARALLELISM); }
/** * Computes a default maximum parallelism from the operator parallelism. This is used in case the user has not * explicitly configured a maximum parallelism to still allow a certain degree of scale-up. * * @param operatorParallelism the operator parallelism as basis for computation. * @return the computed default maximum parallelism. */ public static int computeDefaultMaxParallelism(int operatorParallelism) { checkParallelismPreconditions(operatorParallelism); return Math.min( Math.max( MathUtils.roundUpToPowerOfTwo(operatorParallelism + (operatorParallelism / 2)), DEFAULT_LOWER_BOUND_MAX_PARALLELISM), UPPER_BOUND_MAX_PARALLELISM); }
/** * Computes the range of key-groups that are assigned to a given operator under the given parallelism and maximum * parallelism. * * IMPORTANT: maxParallelism must be <= Short.MAX_VALUE to avoid rounding problems in this method. If we ever want * to go beyond this boundary, this method must perform arithmetic on long values. * * @param maxParallelism Maximal parallelism that the job was initially created with. * @param parallelism The current parallelism under which the job runs. Must be <= maxParallelism. * @param operatorIndex Id of a key-group. 0 <= keyGroupID < maxParallelism. * @return the computed key-group range for the operator. */ public static KeyGroupRange computeKeyGroupRangeForOperatorIndex( int maxParallelism, int parallelism, int operatorIndex) { checkParallelismPreconditions(parallelism); checkParallelismPreconditions(maxParallelism); Preconditions.checkArgument(maxParallelism >= parallelism, "Maximum parallelism must not be smaller than parallelism."); int start = ((operatorIndex * maxParallelism + parallelism - 1) / parallelism); int end = ((operatorIndex + 1) * maxParallelism - 1) / parallelism; return new KeyGroupRange(start, end); }
/** * Computes the range of key-groups that are assigned to a given operator under the given parallelism and maximum * parallelism. * * IMPORTANT: maxParallelism must be <= Short.MAX_VALUE to avoid rounding problems in this method. If we ever want * to go beyond this boundary, this method must perform arithmetic on long values. * * @param maxParallelism Maximal parallelism that the job was initially created with. * @param parallelism The current parallelism under which the job runs. Must be <= maxParallelism. * @param operatorIndex Id of a key-group. 0 <= keyGroupID < maxParallelism. * @return */ public static KeyGroupRange computeKeyGroupRangeForOperatorIndex( int maxParallelism, int parallelism, int operatorIndex) { checkParallelismPreconditions(parallelism); checkParallelismPreconditions(maxParallelism); Preconditions.checkArgument(maxParallelism >= parallelism, "Maximum parallelism must not be smaller than parallelism."); int start = operatorIndex == 0 ? 0 : ((operatorIndex * maxParallelism - 1) / parallelism) + 1; int end = ((operatorIndex + 1) * maxParallelism - 1) / parallelism; return new KeyGroupRange(start, end); }
/** * Computes the range of key-groups that are assigned to a given operator under the given parallelism and maximum * parallelism. * * IMPORTANT: maxParallelism must be <= Short.MAX_VALUE to avoid rounding problems in this method. If we ever want * to go beyond this boundary, this method must perform arithmetic on long values. * * @param maxParallelism Maximal parallelism that the job was initially created with. * @param parallelism The current parallelism under which the job runs. Must be <= maxParallelism. * @param operatorIndex Id of a key-group. 0 <= keyGroupID < maxParallelism. * @return the computed key-group range for the operator. */ public static KeyGroupRange computeKeyGroupRangeForOperatorIndex( int maxParallelism, int parallelism, int operatorIndex) { checkParallelismPreconditions(parallelism); checkParallelismPreconditions(maxParallelism); Preconditions.checkArgument(maxParallelism >= parallelism, "Maximum parallelism must not be smaller than parallelism."); int start = ((operatorIndex * maxParallelism + parallelism - 1) / parallelism); int end = ((operatorIndex + 1) * maxParallelism - 1) / parallelism; return new KeyGroupRange(start, end); }
/** * Computes the range of key-groups that are assigned to a given operator under the given parallelism and maximum * parallelism. * * IMPORTANT: maxParallelism must be <= Short.MAX_VALUE to avoid rounding problems in this method. If we ever want * to go beyond this boundary, this method must perform arithmetic on long values. * * @param maxParallelism Maximal parallelism that the job was initially created with. * @param parallelism The current parallelism under which the job runs. Must be <= maxParallelism. * @param operatorIndex Id of a key-group. 0 <= keyGroupID < maxParallelism. * @return the computed key-group range for the operator. */ public static KeyGroupRange computeKeyGroupRangeForOperatorIndex( int maxParallelism, int parallelism, int operatorIndex) { checkParallelismPreconditions(parallelism); checkParallelismPreconditions(maxParallelism); Preconditions.checkArgument(maxParallelism >= parallelism, "Maximum parallelism must not be smaller than parallelism."); int start = ((operatorIndex * maxParallelism + parallelism - 1) / parallelism); int end = ((operatorIndex + 1) * maxParallelism - 1) / parallelism; return new KeyGroupRange(start, end); }
public ResultPartitionDeploymentDescriptor( IntermediateDataSetID resultId, IntermediateResultPartitionID partitionId, ResultPartitionType partitionType, int numberOfSubpartitions, int maxParallelism, boolean lazyScheduling) { this.resultId = checkNotNull(resultId); this.partitionId = checkNotNull(partitionId); this.partitionType = checkNotNull(partitionType); KeyGroupRangeAssignment.checkParallelismPreconditions(maxParallelism); checkArgument(numberOfSubpartitions >= 1); this.numberOfSubpartitions = numberOfSubpartitions; this.maxParallelism = maxParallelism; this.sendScheduleOrUpdateConsumersMessage = lazyScheduling; }
public ResultPartitionDeploymentDescriptor( IntermediateDataSetID resultId, IntermediateResultPartitionID partitionId, ResultPartitionType partitionType, int numberOfSubpartitions, int maxParallelism, boolean lazyScheduling) { this.resultId = checkNotNull(resultId); this.partitionId = checkNotNull(partitionId); this.partitionType = checkNotNull(partitionType); KeyGroupRangeAssignment.checkParallelismPreconditions(maxParallelism); checkArgument(numberOfSubpartitions >= 1); this.numberOfSubpartitions = numberOfSubpartitions; this.maxParallelism = maxParallelism; this.sendScheduleOrUpdateConsumersMessage = lazyScheduling; }
public ResultPartitionDeploymentDescriptor( IntermediateDataSetID resultId, IntermediateResultPartitionID partitionId, ResultPartitionType partitionType, int numberOfSubpartitions, int maxParallelism, boolean lazyScheduling) { this.resultId = checkNotNull(resultId); this.partitionId = checkNotNull(partitionId); this.partitionType = checkNotNull(partitionType); KeyGroupRangeAssignment.checkParallelismPreconditions(maxParallelism); checkArgument(numberOfSubpartitions >= 1); this.numberOfSubpartitions = numberOfSubpartitions; this.maxParallelism = maxParallelism; this.sendScheduleOrUpdateConsumersMessage = lazyScheduling; }
public ResultPartitionDeploymentDescriptor( IntermediateDataSetID resultId, IntermediateResultPartitionID partitionId, ResultPartitionType partitionType, int numberOfSubpartitions, int maxParallelism, boolean lazyScheduling) { this.resultId = checkNotNull(resultId); this.partitionId = checkNotNull(partitionId); this.partitionType = checkNotNull(partitionType); KeyGroupRangeAssignment.checkParallelismPreconditions(maxParallelism); checkArgument(numberOfSubpartitions >= 1); this.numberOfSubpartitions = numberOfSubpartitions; this.maxParallelism = maxParallelism; this.sendScheduleOrUpdateConsumersMessage = lazyScheduling; }