private PlanNode reportDuplicateId(PlanNode first, PlanNode second) { requireNonNull(first, "first is null"); requireNonNull(second, "second is null"); checkArgument(first.getId().equals(second.getId())); throw new IllegalStateException(format( "Generated plan contains nodes with duplicated id %s: %s and %s", first.getId(), first, second)); } }
private PlanNode reportDuplicateId(PlanNode first, PlanNode second) { requireNonNull(first, "first is null"); requireNonNull(second, "second is null"); checkArgument(first.getId().equals(second.getId())); throw new IllegalStateException(format( "Generated plan contains nodes with duplicated id %s: %s and %s", first.getId(), first, second)); } }
public void updateSource(TaskSource sourceUpdate) { checkLockNotHeld("Can not update sources while holding the driver lock"); checkArgument( sourceOperator.isPresent() && sourceOperator.get().getSourceId().equals(sourceUpdate.getPlanNodeId()), "sourceUpdate is for a plan node that is different from this Driver's source node"); // stage the new updates pendingTaskSourceUpdates.updateAndGet(current -> current == null ? sourceUpdate : current.update(sourceUpdate)); // attempt to get the lock and process the updates we staged above // updates will be processed in close if and only if we got the lock tryWithLock(() -> TRUE); }
public void updateSource(TaskSource sourceUpdate) { checkLockNotHeld("Can not update sources while holding the driver lock"); checkArgument( sourceOperator.isPresent() && sourceOperator.get().getSourceId().equals(sourceUpdate.getPlanNodeId()), "sourceUpdate is for a plan node that is different from this Driver's source node"); // stage the new updates pendingTaskSourceUpdates.updateAndGet(current -> current == null ? sourceUpdate : current.update(sourceUpdate)); // attempt to get the lock and process the updates we staged above // updates will be processed in close if and only if we got the lock tryWithLock(() -> TRUE); }
public TaskSource update(TaskSource source) { checkArgument(planNodeId.equals(source.getPlanNodeId()), "Expected source %s, but got source %s", planNodeId, source.getPlanNodeId()); if (isNewer(source)) { // assure the new source is properly formed // we know that either the new source one has new splits and/or it is marking the source as closed checkArgument(!noMoreSplits || splits.containsAll(source.getSplits()), "Source %s has new splits, but no more splits already set", planNodeId); Set<ScheduledSplit> newSplits = ImmutableSet.<ScheduledSplit>builder() .addAll(splits) .addAll(source.getSplits()) .build(); Set<Lifespan> newNoMoreSplitsForDriverGroup = ImmutableSet.<Lifespan>builder() .addAll(noMoreSplitsForLifespan) .addAll(source.getNoMoreSplitsForLifespan()) .build(); return new TaskSource( planNodeId, newSplits, newNoMoreSplitsForDriverGroup, source.isNoMoreSplits()); } else { // the specified source is older than this one return this; } }
public TaskSource update(TaskSource source) { checkArgument(planNodeId.equals(source.getPlanNodeId()), "Expected source %s, but got source %s", planNodeId, source.getPlanNodeId()); if (isNewer(source)) { // assure the new source is properly formed // we know that either the new source one has new splits and/or it is marking the source as closed checkArgument(!noMoreSplits || splits.containsAll(source.getSplits()), "Source %s has new splits, but no more splits already set", planNodeId); Set<ScheduledSplit> newSplits = ImmutableSet.<ScheduledSplit>builder() .addAll(splits) .addAll(source.getSplits()) .build(); Set<Lifespan> newNoMoreSplitsForDriverGroup = ImmutableSet.<Lifespan>builder() .addAll(noMoreSplitsForLifespan) .addAll(source.getNoMoreSplitsForLifespan()) .build(); return new TaskSource( planNodeId, newSplits, newNoMoreSplitsForDriverGroup, source.isNoMoreSplits()); } else { // the specified source is older than this one return this; } }
public StatsCalculatorAssertion withSourceStats(PlanNodeId planNodeId, PlanNodeStatsEstimate sourceStats) { PlanNode sourceNode = PlanNodeSearcher.searchFrom(planNode).where(node -> node.getId().equals(planNodeId)).findOnlyElement(); sourcesStats.put(sourceNode, sourceStats); return this; }
public StatsCalculatorAssertion withSourceStats(PlanNodeId planNodeId, PlanNodeStatsEstimate sourceStats) { PlanNode sourceNode = PlanNodeSearcher.searchFrom(planNode).where(node -> node.getId().equals(planNodeId)).findOnlyElement(); sourcesStats.put(sourceNode, sourceStats); return this; }
@Override public PlanNodeStats mergeWith(PlanNodeStats other) { checkArgument(planNodeId.equals(other.getPlanNodeId()), "planNodeIds do not match. %s != %s", planNodeId, other.getPlanNodeId()); long planNodeInputPositions = this.planNodeInputPositions + other.planNodeInputPositions; DataSize planNodeInputDataSize = succinctBytes(this.planNodeInputDataSize.toBytes() + other.planNodeInputDataSize.toBytes()); long planNodeOutputPositions = this.planNodeOutputPositions + other.planNodeOutputPositions; DataSize planNodeOutputDataSize = succinctBytes(this.planNodeOutputDataSize.toBytes() + other.planNodeOutputDataSize.toBytes()); Map<String, OperatorInputStats> operatorInputStats = mergeMaps(this.operatorInputStats, other.operatorInputStats, OperatorInputStats::merge); Map<String, OperatorHashCollisionsStats> operatorHashCollisionsStats = mergeMaps(this.operatorHashCollisionsStats, other.operatorHashCollisionsStats, OperatorHashCollisionsStats::merge); Optional<WindowOperatorStats> windowNodeStats = Mergeable.merge(this.windowOperatorStats, other.windowOperatorStats); return new PlanNodeStats( planNodeId, new Duration(planNodeScheduledTime.toMillis() + other.getPlanNodeScheduledTime().toMillis(), MILLISECONDS), new Duration(planNodeCpuTime.toMillis() + other.getPlanNodeCpuTime().toMillis(), MILLISECONDS), planNodeInputPositions, planNodeInputDataSize, planNodeOutputPositions, planNodeOutputDataSize, operatorInputStats, operatorHashCollisionsStats, windowNodeStats); } }
@Override public PlanNodeStats mergeWith(PlanNodeStats other) { checkArgument(planNodeId.equals(other.getPlanNodeId()), "planNodeIds do not match. %s != %s", planNodeId, other.getPlanNodeId()); long planNodeInputPositions = this.planNodeInputPositions + other.planNodeInputPositions; DataSize planNodeInputDataSize = succinctBytes(this.planNodeInputDataSize.toBytes() + other.planNodeInputDataSize.toBytes()); long planNodeOutputPositions = this.planNodeOutputPositions + other.planNodeOutputPositions; DataSize planNodeOutputDataSize = succinctBytes(this.planNodeOutputDataSize.toBytes() + other.planNodeOutputDataSize.toBytes()); Map<String, OperatorInputStats> operatorInputStats = mergeMaps(this.operatorInputStats, other.operatorInputStats, OperatorInputStats::merge); Map<String, OperatorHashCollisionsStats> operatorHashCollisionsStats = mergeMaps(this.operatorHashCollisionsStats, other.operatorHashCollisionsStats, OperatorHashCollisionsStats::merge); Optional<WindowOperatorStats> windowNodeStats = Mergeable.merge(this.windowOperatorStats, other.windowOperatorStats); return new PlanNodeStats( planNodeId, new Duration(planNodeScheduledTime.toMillis() + other.getPlanNodeScheduledTime().toMillis(), MILLISECONDS), new Duration(planNodeCpuTime.toMillis() + other.getPlanNodeCpuTime().toMillis(), MILLISECONDS), planNodeInputPositions, planNodeInputDataSize, planNodeOutputPositions, planNodeOutputDataSize, operatorInputStats, operatorHashCollisionsStats, windowNodeStats); } }