@Override public String executionStatus() { return this.status.getState().name(); }
@Override public String executionStatus() { return this.status.getState().name(); }
/** * Try to get most the data required from dagClient in the constructor itself so that even after * the tez job has finished this object can be used for later use.s */ TezProgressMonitor(DAGClient dagClient, DAGStatus status, List<BaseWork> topSortedWork, Map<String, Progress> progressMap, SessionState.LogHelper console, long executionStartTime) throws IOException, TezException { this.status = status; this.topSortedWork = topSortedWork; this.console = console; this.executionStartTime = executionStartTime; for (Map.Entry<String, Progress> entry : progressMap.entrySet()) { String vertexName = entry.getKey(); progressCountsMap.put(vertexName, new VertexProgress(entry.getValue(), status.getState())); try { vertexStatusMap.put(vertexName, dagClient.getVertexStatus(vertexName, null)); } catch (IOException e) { // best attempt, shouldn't really kill DAG for this } } }
/** * Try to get most the data required from dagClient in the constructor itself so that even after * the tez job has finished this object can be used for later use.s */ TezProgressMonitor(DAGClient dagClient, DAGStatus status, Map<String, BaseWork> workMap, Map<String, Progress> progressMap, SessionState.LogHelper console, long executionStartTime) throws IOException, TezException { this.status = status; this.workMap = workMap; this.console = console; this.executionStartTime = executionStartTime; for (Map.Entry<String, Progress> entry : progressMap.entrySet()) { String vertexName = entry.getKey(); progressCountsMap.put(vertexName, new VertexProgress(entry.getValue(), status.getState())); try { vertexStatusMap.put(vertexName, dagClient.getVertexStatus(vertexName, null)); } catch (IOException e) { // best attempt, shouldn't really kill DAG for this } } }
@Test public void setupInternalStateOnObjectCreation() throws IOException, TezException { when(dagStatus.getState()).thenReturn(DAGStatus.State.RUNNING); when(dagClient.getVertexStatus(eq(MAPPER), anySet())).thenReturn(succeeded); when(dagClient.getVertexStatus(eq(REDUCER), anySet())).thenReturn(running); TezProgressMonitor monitor = new TezProgressMonitor(dagClient, dagStatus, new ArrayList<BaseWork>(), progressMap(), console, Long.MAX_VALUE); verify(dagClient).getVertexStatus(eq(MAPPER), isNull(Set.class)); verify(dagClient).getVertexStatus(eq(REDUCER), isNull(Set.class)); verifyNoMoreInteractions(dagClient); assertThat(monitor.vertexStatusMap.keySet(), hasItems(MAPPER, REDUCER)); assertThat(monitor.vertexStatusMap.get(MAPPER), is(sameInstance(succeeded))); assertThat(monitor.vertexStatusMap.get(REDUCER), is(sameInstance(running))); assertThat(monitor.progressCountsMap.keySet(), hasItems(MAPPER, REDUCER)); TezProgressMonitor.VertexProgress expectedMapperState = new TezProgressMonitor.VertexProgress(2, 1, 3, 4, 5, DAGStatus.State.RUNNING); assertThat(monitor.progressCountsMap.get(MAPPER), is(equalTo(expectedMapperState))); TezProgressMonitor.VertexProgress expectedReducerState = new TezProgressMonitor.VertexProgress(3, 2, 1, 0, 1, DAGStatus.State.RUNNING); assertThat(monitor.progressCountsMap.get(REDUCER), is(equalTo(expectedReducerState))); }
DAGStatus.State state = status.getState();
DAGStatus.State state = status.getState();
@Override public synchronized float getSetupProgress() { if (dagStatus.getState() == DAGStatus.State.RUNNING) { return 1.0f; } return 0.0f; }
public boolean isCompleted() { State state = getState(); return (state == State.SUCCEEDED || state == State.FAILED || state == State.KILLED || state == State.ERROR); }
@Override public String toString() { StringBuilder sb = new StringBuilder(); sb.append("status=" + getState() + ", progress=" + getDAGProgress() + ", diagnostics=" + StringUtils.join(getDiagnostics(), LINE_SEPARATOR) + ", counters=" + (getDAGCounters() == null ? "null" : getDAGCounters().toString())); return sb.toString(); }
@Override public synchronized float getMapProgress() { if(dagStatus.getVertexProgress() != null) { return getProgress(MultiStageMRConfigUtil.getInitialMapVertexName()); } if (dagStatus.getState() == DAGStatus.State.SUCCEEDED) { return 1.0f; } return 0.0f; }
@Override public synchronized float getReduceProgress() { if(dagStatus.getVertexProgress() != null) { return getProgress(MultiStageMRConfigUtil.getFinalReduceVertexName()); } if (dagStatus.getState() == DAGStatus.State.SUCCEEDED) { return 1.0f; } return 0.0f; }
private DAGStatus.State getDagStatusState() { DAGStatus dagStatus = getDagStatus(); if( dagStatus == null ) { flowStep.logWarn( "getDagStatus returned null" ); return null; } DAGStatus.State state = dagStatus.getState(); if( state == null ) flowStep.logWarn( "dagStatus#getState returned null" ); return state; }
private DAGStatus.State getDagStatusState() { DAGStatus dagStatus = getDagStatus(); if( dagStatus == null ) { flowStep.logWarn( "getDagStatus returned null" ); return null; } DAGStatus.State state = dagStatus.getState(); if( state == null ) flowStep.logWarn( "dagStatus#getState returned null" ); return state; }
public void notifyUpdate() { DAGStatus dagStatus = runningJob.getDAGStatus(); if (dagStatus != null && dagStatus.getState() == DAGStatus.State.RUNNING) { // Emit notification when the job has progressed more than 1%, // or every 20 seconds int currProgress = Math.round(runningJob.getDAGProgress() * 100f); if (currProgress - prevProgress >= 1 || count % 100 == 0) { tezScriptState.dagProgressNotification(runningJob.getName(), -1, currProgress); tezScriptState.emitProgressUpdatedNotification((currProgress + (100 * processedDAGS))/totalDAGs); prevProgress = currProgress; } count++; } // TODO: Add new vertex tracking methods to PigTezProgressNotificationListener // and emit notifications for individual vertex start, progress and completion }
@Override public boolean equals(Object obj) { // Source explicitly exclude from equals if (obj instanceof DAGStatus){ DAGStatus other = (DAGStatus)obj; return getState() == other.getState() && getDiagnostics().equals(other.getDiagnostics()) && getDAGProgress().equals(other.getDAGProgress()) && getVertexProgress().equals(other.getVertexProgress()) && ((getDAGCounters() == null && other.getDAGCounters() == null) || getDAGCounters().equals(other.getDAGCounters())); } return false; }
protected void dumpDebugInfo() { DAGStatus dagStatus = getDagStatus(); if( dagStatus == null ) return; flowStep.logWarn( "Tez DAG " + dagId + " state at " + dagStatus.getState() ); flowStep.logWarn( "failure info: " ); for( String diagLine : dagStatus.getDiagnostics() ) flowStep.logWarn( diagLine ); }
protected void dumpDebugInfo() { DAGStatus dagStatus = getDagStatus(); if( dagStatus == null ) return; flowStep.logWarn( "Tez DAG " + dagId + " state at " + dagStatus.getState() ); flowStep.logWarn( "failure info: " ); for( String diagLine : dagStatus.getDiagnostics() ) flowStep.logWarn( diagLine ); }