public boolean isTrivial(){ Integer last = getLast(); if(last != null && last <= PrelUtil.getPlannerSettings(getCluster()).getSliceTarget() ){ return true; } else { return false; } }
protected static boolean isSingleton(RelOptRuleCall call) { PlannerSettings settings = PrelUtil.getPlannerSettings(call.getPlanner()); if (settings.isSingleMode()) { return true; } RelNode child = call.rel(0).getInputs().get(0); // if small input, then singleton return call.getMetadataQuery().getRowCount(child) < settings.getSliceTarget(); }
public SimpleParallelizer(QueryContext context, AttemptObserver observer, Collection<NodeEndpoint> activeEndpoints) { this.queryContext = context; OptionManager optionManager = context.getOptions(); long sliceTarget = context.getPlannerSettings().getSliceTarget(); this.parallelizationThreshold = sliceTarget > 0 ? sliceTarget : 1; final long configuredMaxWidthPerNode = context.getClusterResourceInformation().getAverageExecutorCores(optionManager); final double maxWidthFactor = context.getWorkStatsProvider().get().getMaxWidthFactor(); this.maxWidthPerNode = (int) Math.max(1, configuredMaxWidthPerNode * maxWidthFactor); if (logger.isDebugEnabled() && maxWidthFactor < 1) { final float clusterLoad = context.getWorkStatsProvider().get().getClusterLoad(); logger.debug("Cluster load {} exceeded cutoff, max_width_factor = {}. current max_width = {}", clusterLoad, maxWidthFactor, this.maxWidthPerNode); } this.executionMap = new ExecutionNodeMap(Optional.ofNullable(activeEndpoints).orElse(context.getActiveEndpoints())); this.maxGlobalWidth = (int) optionManager.getOption(ExecConstants.MAX_WIDTH_GLOBAL); this.affinityFactor = optionManager.getOption(ExecConstants.AFFINITY_FACTOR); this.useNewAssignmentCreator = !optionManager.getOption(ExecConstants.OLD_ASSIGNMENT_CREATOR); this.assignmentCreatorBalanceFactor = optionManager.getOption(ExecConstants.ASSIGNMENT_CREATOR_BALANCE_FACTOR); this.observer = observer; this.fragmentCodec = FragmentCodec.valueOf(optionManager.getOption(ExecConstants.FRAGMENT_CODEC).toUpperCase()); }
private static RelTraitSet traits( RelOptCluster cluster, double rowCount, int splitCount, RelTraitSet traitSet) { PlannerSettings settings = PrelUtil.getPlannerSettings(cluster.getPlanner()); boolean smallInput = rowCount < (double)settings.getSliceTarget(); DistributionTrait distribution; if(settings.isMultiPhaseAggEnabled() && !settings.isSingleMode() && !smallInput && splitCount > 1) { distribution = DistributionTrait.ANY; } else { distribution = DistributionTrait.SINGLETON; } return traitSet.plus(distribution); }
final long maxWidthPerNode = plannerSettings.getNumCoresPerExecutor(); final long maxWidthPerQuery = plannerSettings.getOptions().getOption(ExecConstants.MAX_WIDTH_GLOBAL); final long sliceTarget = plannerSettings.getSliceTarget(); final double minFactor = Doubles.min(leftRowCount * 1.0 / sliceTarget, numEndPoints * maxWidthPerNode, maxWidthPerQuery); final boolean enableBroadCast = (minFactor * broadcastFactor < leftRowCount);
long targetSliceSize = config.getContext().getPlannerSettings().getSliceTarget(); phyRelNode = ExcessiveExchangeIdentifier.removeExcessiveEchanges(phyRelNode, targetSliceSize);