private static URI findCancelableLeafStage(StageInfo stage) { // if this stage is already done, we can't cancel it if (stage.getState().isDone()) { return null; } // attempt to find a cancelable sub stage // check in reverse order since build side of a join will be later in the list for (StageInfo subStage : Lists.reverse(stage.getSubStages())) { URI leafStage = findCancelableLeafStage(subStage); if (leafStage != null) { return leafStage; } } // no matching sub stage, so return this stage return stage.getSelf(); }
private StageInfo buildStageInfo(StageId stageId, Map<StageId, StageInfo> stageInfos) { StageInfo parent = stageInfos.get(stageId); checkArgument(parent != null, "No stageInfo for %s", parent); List<StageInfo> childStages = stageLinkages.get(stageId).getChildStageIds().stream() .map(childStageId -> buildStageInfo(childStageId, stageInfos)) .collect(toImmutableList()); if (childStages.isEmpty()) { return parent; } return new StageInfo( parent.getStageId(), parent.getState(), parent.getSelf(), parent.getPlan(), parent.getTypes(), parent.getStageStats(), parent.getTasks(), childStages, parent.getFailureCause()); }
private static void assertState(StageStateMachine stateMachine, StageState expectedState) { assertEquals(stateMachine.getStageId(), STAGE_ID); assertEquals(stateMachine.getLocation(), LOCATION); assertSame(stateMachine.getSession(), TEST_SESSION); StageInfo stageInfo = stateMachine.getStageInfo(ImmutableList::of); assertEquals(stageInfo.getStageId(), STAGE_ID); assertEquals(stageInfo.getSelf(), LOCATION); assertEquals(stageInfo.getSubStages(), ImmutableList.of()); assertEquals(stageInfo.getTasks(), ImmutableList.of()); assertEquals(stageInfo.getTypes(), ImmutableList.of(VARCHAR)); assertSame(stageInfo.getPlan(), PLAN_FRAGMENT); assertEquals(stateMachine.getState(), expectedState); assertEquals(stageInfo.getState(), expectedState); if (expectedState == StageState.FAILED) { ExecutionFailureInfo failure = stageInfo.getFailureCause(); assertEquals(failure.getMessage(), FAILED_CAUSE.getMessage()); assertEquals(failure.getType(), FAILED_CAUSE.getClass().getName()); } else { assertNull(stageInfo.getFailureCause()); } }
return finalStageInfo.get() .getStageStats() .toBasicStageStats(finalStageInfo.get().getState());
Optional<StageInfo> prunedOutputStage = queryInfo.getOutputStage().map(outputStage -> new StageInfo( outputStage.getStageId(), outputStage.getState(), outputStage.getSelf(),
totalCpuTime += stageStats.getTotalCpuTime().roundTo(MILLISECONDS); totalBlockedTime += stageStats.getTotalBlockedTime().roundTo(MILLISECONDS); if (!stageInfo.getState().isDone()) { fullyBlocked &= stageStats.isFullyBlocked(); blockedReasons.addAll(stageStats.getBlockedReasons());
private static URI findCancelableLeafStage(StageInfo stage) { // if this stage is already done, we can't cancel it if (stage.getState().isDone()) { return null; } // attempt to find a cancelable sub stage // check in reverse order since build side of a join will be later in the list for (StageInfo subStage : Lists.reverse(stage.getSubStages())) { URI leafStage = findCancelableLeafStage(subStage); if (leafStage != null) { return leafStage; } } // no matching sub stage, so return this stage return stage.getSelf(); }
private static boolean allOutputBuffersCreated(StageInfo outputStage) { StageState stageState = outputStage.getState(); // if the stage is already done, then there will be no more buffers if (stageState.isDone()) { return true; } // have all stage tasks been scheduled? if (stageState == StageState.PLANNED || stageState == StageState.SCHEDULING) { return false; } // have all tasks finished adding buffers return outputStage.getTasks().stream() .allMatch(taskInfo -> !taskInfo.getOutputBuffers().getState().canAddBuffers()); }
private StageInfo buildStageInfo(StageId stageId, Map<StageId, StageInfo> stageInfos) { StageInfo parent = stageInfos.get(stageId); checkArgument(parent != null, "No stageInfo for %s", parent); List<StageInfo> childStages = stageLinkages.get(stageId).getChildStageIds().stream() .map(childStageId -> buildStageInfo(childStageId, stageInfos)) .collect(toImmutableList()); if (childStages.isEmpty()) { return parent; } return new StageInfo( parent.getStageId(), parent.getState(), parent.getSelf(), parent.getPlan(), parent.getTypes(), parent.getStageStats(), parent.getTasks(), childStages, parent.getFailureCause()); }
private static void assertState(StageStateMachine stateMachine, StageState expectedState) { assertEquals(stateMachine.getStageId(), STAGE_ID); assertEquals(stateMachine.getLocation(), LOCATION); assertSame(stateMachine.getSession(), TEST_SESSION); StageInfo stageInfo = stateMachine.getStageInfo(ImmutableList::of, ImmutableList::of); assertEquals(stageInfo.getStageId(), STAGE_ID); assertEquals(stageInfo.getSelf(), LOCATION); assertEquals(stageInfo.getSubStages(), ImmutableList.of()); assertEquals(stageInfo.getTasks(), ImmutableList.of()); assertEquals(stageInfo.getTypes(), ImmutableList.of(VARCHAR)); assertSame(stageInfo.getPlan(), PLAN_FRAGMENT); assertEquals(stateMachine.getState(), expectedState); assertEquals(stageInfo.getState(), expectedState); if (expectedState == StageState.FAILED) { ExecutionFailureInfo failure = stageInfo.getFailureCause(); assertEquals(failure.getMessage(), FAILED_CAUSE.getMessage()); assertEquals(failure.getType(), FAILED_CAUSE.getClass().getName()); } else { assertNull(stageInfo.getFailureCause()); } }
private synchronized void updateExchangeClient(StageInfo outputStage) { // add any additional output locations if (!outputStage.getState().isDone()) { for (TaskInfo taskInfo : outputStage.getTasks()) { SharedBufferInfo outputBuffers = taskInfo.getOutputBuffers(); List<BufferInfo> buffers = outputBuffers.getBuffers(); if (buffers.isEmpty() || outputBuffers.getState().canAddBuffers()) { // output buffer has not been created yet continue; } Preconditions.checkState(buffers.size() == 1, "Expected a single output buffer for task %s, but found %s", taskInfo.getTaskId(), buffers); TaskId bufferId = Iterables.getOnlyElement(buffers).getBufferId(); URI uri = uriBuilderFrom(taskInfo.getSelf()).appendPath("results").appendPath(bufferId.toString()).build(); exchangeClient.addLocation(uri); } } if (allOutputBuffersCreated(outputStage)) { exchangeClient.noMoreLocations(); } }
totalUserTime += stageStats.getTotalUserTime().roundTo(NANOSECONDS); totalBlockedTime += stageStats.getTotalBlockedTime().roundTo(NANOSECONDS); if (!stageInfo.getState().isDone()) { fullyBlocked &= stageStats.isFullyBlocked(); blockedReasons.addAll(stageStats.getBlockedReasons());
queryInfo.getOutputStage().getState(), queryInfo.getOutputStage().getSelf(),