public void waitForStageState(Stage stage, StageState expected, int delay) throws Exception { int i = 0; while (stage == null || stage.getSynchronizedState() != expected) { try { Thread.sleep(delay); } catch (InterruptedException e) { } if (++i > 200) { throw new IOException("Timed out waiting"); } } }
if (LOG.isDebugEnabled()) { LOG.debug("Processing " + event.getStageId() + " of type " + event.getType() + ", preState=" + getSynchronizedState()); StageState oldState = getSynchronizedState(); try { getStateMachine().doTransition(event.getType(), event); stageState = getSynchronizedState(); } catch (InvalidStateTransitonException e) { LOG.error("Can't handle this event at current state" + ", eventType:" + event.getType().name() + ", oldState:" + oldState.name() + ", nextState:" + getSynchronizedState().name() , e); eventHandler.handle(new StageEvent(getId(), if (oldState != getSynchronizedState()) { LOG.debug(getId() + " Stage Transitioned from " + oldState + " to " + getSynchronizedState());
@Override public void run() { while (stage.getSynchronizedState() == StageState.FINALIZING && !Thread.interrupted()) { long elapsedTime = System.currentTimeMillis() - stage.lastContactTime; if (elapsedTime > 120 * 1000) { stage.stopFinalization(); LOG.error(stage.getId() + ": Timed out while receiving intermediate reports: " + elapsedTime + " ms, report:" + stage.completedShuffleTasks.get() + "/" + stage.succeededObjectCount); stage.getEventHandler().handle(new StageEvent(stage.getId(), StageEventType.SQ_FAILED)); } synchronized (this) { try { this.wait(1 * 1000); } catch (InterruptedException e) { } } } } });
if (LOG.isDebugEnabled()) { LOG.debug("Processing " + event.getStageId() + " of type " + event.getType() + ", preState=" + getSynchronizedState()); StageState oldState = getSynchronizedState(); try { getStateMachine().doTransition(event.getType(), event); stageState = getSynchronizedState(); } catch (InvalidStateTransitonException e) { LOG.error("Can't handle this event at current state" + ", eventType:" + event.getType().name() + ", oldState:" + oldState.name() + ", nextState:" + getSynchronizedState().name() , e); eventHandler.handle(new StageEvent(getId(), if (oldState != getSynchronizedState()) { LOG.debug(getId() + " Stage Transitioned from " + oldState + " to " + getSynchronizedState());
@Override public void run() { while (stage.getSynchronizedState() == StageState.FINALIZING && !Thread.interrupted()) { long elapsedTime = System.currentTimeMillis() - stage.lastContactTime; if (elapsedTime > 120 * 1000) { stage.stopFinalization(); LOG.error(stage.getId() + ": Timed out while receiving intermediate reports: " + elapsedTime + " ms, report:" + stage.completedShuffleTasks.get() + "/" + stage.succeededObjectCount); stage.getEventHandler().handle(new StageEvent(stage.getId(), StageEventType.SQ_FAILED)); } synchronized (this) { try { this.wait(1 * 1000); } catch (InterruptedException e) { } } } } });
public static long getInputVolume(MasterPlan masterPlan, QueryMasterTask.QueryMasterTaskContext context, ExecutionBlock execBlock) { if (masterPlan.isLeaf(execBlock)) { ScanNode[] outerScans = execBlock.getScanNodes(); long maxVolume = 0; for (ScanNode eachScanNode: outerScans) { TableStats stat = context.getTableDesc(eachScanNode).getStats(); if (stat.getNumBytes() > maxVolume) { maxVolume = stat.getNumBytes(); } } return maxVolume; } else { long aggregatedVolume = 0; for (ExecutionBlock childBlock : masterPlan.getChilds(execBlock)) { Stage stage = context.getStage(childBlock.getId()); if (stage == null || stage.getSynchronizedState() != StageState.SUCCEEDED) { aggregatedVolume += getInputVolume(masterPlan, context, childBlock); } else { aggregatedVolume += stage.getResultStats().getNumBytes(); } } return aggregatedVolume; } }
new StageEvent(stage.getId(), StageEventType.SQ_STAGE_COMPLETED)); } else { if(stage.getSynchronizedState() == StageState.INITED) { stage.eventHandler.handle(new StageEvent(stage.getId(), StageEventType.SQ_START)); stage.taskScheduler.start();
public static long getInputVolume(MasterPlan masterPlan, QueryMasterTask.QueryMasterTaskContext context, ExecutionBlock execBlock) { if (masterPlan.isLeaf(execBlock)) { ScanNode[] outerScans = execBlock.getScanNodes(); long maxVolume = 0; for (ScanNode eachScanNode: outerScans) { TableStats stat = context.getTableDesc(eachScanNode).getStats(); if (stat.getNumBytes() > maxVolume) { maxVolume = stat.getNumBytes(); } } return maxVolume; } else { long aggregatedVolume = 0; for (ExecutionBlock childBlock : masterPlan.getChilds(execBlock)) { Stage stage = context.getStage(childBlock.getId()); if (stage == null || stage.getSynchronizedState() != StageState.SUCCEEDED) { aggregatedVolume += getInputVolume(masterPlan, context, childBlock); } else { aggregatedVolume += stage.getResultStats().getNumBytes(); } } return aggregatedVolume; } }
new StageEvent(stage.getId(), StageEventType.SQ_STAGE_COMPLETED)); } else { if(stage.getSynchronizedState() == StageState.INITED) { stage.taskScheduler.start(); stage.eventHandler.handle(new StageEvent(stage.getId(), StageEventType.SQ_START));
Stage lastStage = stages.get(stages.size() - 1); assertEquals(StageState.KILLED, lastStage.getSynchronizedState());