private void updateProgress(long bytesProcessed) { totalBytesProcessed += bytesProcessed; if (progPerByte > 0) { mergeProgress.set(totalBytesProcessed * progPerByte); } }
mapOutputFile.getOutputIndexFileForWriteInVolume(filename[0]), job); sortPhase.complete(); return; finalOut.close(); sortPhase.complete(); return; sortPhase.addPhases(partitions); // Divide sort phase into sub-phases new Path(mapId.toString()), job.getOutputKeyComparator(), reporter, sortSegments, null, spilledRecordsCounter, sortPhase.phase(), TaskType.MAP); sortPhase.startNextPhase();
public void informReduceProgress() { reducePhase.set(super.in.getProgress().getProgress()); // update progress reporter.progress(); } }
/** Computes progress in this node. */ private synchronized float getInternal() { int phaseCount = phases.size(); if (phaseCount != 0) { float subProgress = 0.0f; float progressFromCurrentPhase = 0.0f; if (currentPhase < phaseCount) { subProgress = phase().getInternal(); progressFromCurrentPhase = getProgressWeightage(currentPhase) * subProgress; } float progressFromCompletedPhases = 0.0f; if (fixedWeightageForAllPhases) { // same progress weightage for each phase progressFromCompletedPhases = progressPerPhase * currentPhase; } else { for (int i = 0; i < currentPhase; i++) { // progress weightages of phases could be different. Add them progressFromCompletedPhases += getProgressWeightage(i); } } return progressFromCompletedPhases + progressFromCurrentPhase; } else { return progress; } }
public boolean next() throws IOException { boolean ret = rawIter.next(); reporter.setProgress(rawIter.getProgress().getProgress()); return ret; } };
/** Completes this node, moving the parent node to its next child. */ public void complete() { // we have to traverse up to our parent, so be careful about locking. Progress myParent; synchronized(this) { progress = 1.0f; myParent = parent; } if (myParent != null) { // this will synchronize on the parent, so we make sure we release // our lock before getting the parent's, since we're traversing // against the normal traversal direction used by get() or toString(). // We don't need transactional semantics, so we're OK doing this. myParent.startNextPhase(); } }
public void setStatus(String status) { taskProgress.setStatus(normalizeStatus(status, conf)); // indicate that progress update needs to be sent setProgressFlag(); } public void setProgress(float progress) {
sortPhase.addPhases(partitions); // Divide sort phase into sub-phases Merger.considerFinalMergeForProgress(); new Path(mapId.toString()), job.getOutputKeyComparator(), reporter, sortSegments, null, spilledRecordsCounter, sortPhase.phase()); sortPhase.startNextPhase();
doReturn(mof).when(mockTask).getMapOutputFile(); doReturn(attemptId).when(mockTask).getTaskID(); doReturn(new Progress()).when(mockTask).getSortPhase(); TaskReporter mockReporter = mock(TaskReporter.class); doReturn(new Counter()).when(mockReporter).getCounter(
@Override public Progress getProgress() { progress.complete(); return progress; }
@Override public String toString() { StringBuilder result = new StringBuilder(); toString(result); return result.toString(); }
Progress progress = new Progress(); scheduler.tipFailed(taskId1); Assert.assertEquals("Progress should be 0.5", 0.5f, progress.getProgress(), 0.0f); Assert.assertFalse(scheduler.waitUntilDone(1)); Assert.assertEquals("Progress should be 1.0", 1.0f, progress.getProgress(), 0.0f); Assert.assertTrue(scheduler.waitUntilDone(1));
Progress progress = new Progress(); ShuffleSchedulerImpl<K, V> scheduler = new ShuffleSchedulerImpl<K, V>(job, status, null, null, progress, context.getShuffledMapsCounter(), Assert.assertEquals(copyMessage(1, 1, 1), progress.toString()); Assert.assertEquals(copyMessage(2, 1, 1), progress.toString()); Assert.assertEquals(copyMessage(3, 2, 2), progress.toString()); Assert.assertEquals(copyMessage(4, 0.5, 1), progress.toString()); Assert.assertEquals(copyMessage(5, 1, 1), progress.toString()); Assert.assertEquals(copyMessage(6, 1, 1), progress.toString()); Assert.assertEquals(copyMessage(7, 1, 1), progress.toString()); Assert.assertEquals(copyMessage(8, 0.5, 1), progress.toString()); Assert.assertEquals(copyMessage(9, 1, 1), progress.toString()); Assert.assertEquals(copyMessage(10, 1, 2), progress.toString());