/** * The crux of the matter... */ private void doCopy(Set<TaskAttemptID> maps) throws IOException { Iterator<TaskAttemptID> iter = maps.iterator(); while (iter.hasNext()) { TaskAttemptID map = iter.next(); LOG.debug("LocalFetcher " + id + " going to fetch: " + map); if (copyMapOutput(map)) { // Successful copy. Remove this from our worklist. iter.remove(); } else { // We got back a WAIT command; go back to the outer loop // and block for InMemoryMerge. break; } } }
public LocalFetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl<K, V> scheduler, MergeManager<K,V> merger, Reporter reporter, ShuffleClientMetrics metrics, ExceptionReporter exceptionReporter, SecretKey shuffleKey, Map<TaskAttemptID, MapOutputFile> localMapFiles) { super(job, reduceId, scheduler, merger, reporter, metrics, exceptionReporter, shuffleKey); this.job = job; this.localMapFiles = localMapFiles; setName("localfetcher#" + id); setDaemon(true); }
public void run() { // Create a worklist of task attempts to work over. Set<TaskAttemptID> maps = new HashSet<TaskAttemptID>(); for (TaskAttemptID map : localMapFiles.keySet()) { maps.add(map); } while (maps.size() > 0) { try { // If merge is on, block merger.waitForResource(); metrics.threadBusy(); // Copy as much as is possible. doCopy(maps); metrics.threadFree(); } catch (InterruptedException ie) { } catch (Throwable t) { exceptionReporter.reportException(t); } } }
Fetcher<K,V>[] fetchers = new Fetcher[numFetchers]; if (isLocal) { fetchers[0] = new LocalFetcher<K, V>(jobConf, reduceId, scheduler, merger, reporter, metrics, this, reduceTask.getShuffleSecret(), localMapFiles);
public void run() { // Create a worklist of task attempts to work over. Set<TaskAttemptID> maps = new HashSet<TaskAttemptID>(); for (TaskAttemptID map : localMapFiles.keySet()) { maps.add(map); } while (maps.size() > 0) { try { // If merge is on, block merger.waitForResource(); metrics.threadBusy(); // Copy as much as is possible. doCopy(maps); metrics.threadFree(); } catch (InterruptedException ie) { } catch (Throwable t) { exceptionReporter.reportException(t); } } }
Fetcher<K,V>[] fetchers = new Fetcher[numFetchers]; if (isLocal) { fetchers[0] = new LocalFetcher<K, V>(jobConf, reduceId, scheduler, merger, reporter, metrics, this, reduceTask.getShuffleSecret(), localMapFiles);
public LocalFetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl<K, V> scheduler, MergeManager<K,V> merger, Reporter reporter, ShuffleClientMetrics metrics, ExceptionReporter exceptionReporter, SecretKey shuffleKey, Map<TaskAttemptID, MapOutputFile> localMapFiles) { super(job, reduceId, scheduler, merger, reporter, metrics, exceptionReporter, shuffleKey); this.job = job; this.localMapFiles = localMapFiles; setName("localfetcher#" + id); setDaemon(true); }
/** * The crux of the matter... */ private void doCopy(Set<TaskAttemptID> maps) throws IOException { Iterator<TaskAttemptID> iter = maps.iterator(); while (iter.hasNext()) { TaskAttemptID map = iter.next(); LOG.debug("LocalFetcher " + id + " going to fetch: " + map); if (copyMapOutput(map)) { // Successful copy. Remove this from our worklist. iter.remove(); } else { // We got back a WAIT command; go back to the outer loop // and block for InMemoryMerge. break; } } }
public void run() { // Create a worklist of task attempts to work over. Set<TaskAttemptID> maps = new HashSet<TaskAttemptID>(); for (TaskAttemptID map : localMapFiles.keySet()) { maps.add(map); } while (maps.size() > 0) { try { // If merge is on, block merger.waitForResource(); metrics.threadBusy(); // Copy as much as is possible. doCopy(maps); metrics.threadFree(); } catch (InterruptedException ie) { } catch (Throwable t) { exceptionReporter.reportException(t); } } }
Fetcher<K,V>[] fetchers = new Fetcher[numFetchers]; if (isLocal) { fetchers[0] = new LocalFetcher<K, V>(jobConf, reduceId, scheduler, merger, reporter, metrics, this, reduceTask.getShuffleSecret(), localMapFiles);
public LocalFetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl<K, V> scheduler, MergeManager<K,V> merger, Reporter reporter, ShuffleClientMetrics metrics, ExceptionReporter exceptionReporter, SecretKey shuffleKey, Map<TaskAttemptID, MapOutputFile> localMapFiles) { super(job, reduceId, scheduler, merger, reporter, metrics, exceptionReporter, shuffleKey); this.job = job; this.localMapFiles = localMapFiles; setName("localfetcher#" + id); setDaemon(true); }
/** * The crux of the matter... */ private void doCopy(Set<TaskAttemptID> maps) throws IOException { Iterator<TaskAttemptID> iter = maps.iterator(); while (iter.hasNext()) { TaskAttemptID map = iter.next(); LOG.debug("LocalFetcher " + id + " going to fetch: " + map); if (copyMapOutput(map)) { // Successful copy. Remove this from our worklist. iter.remove(); } else { // We got back a WAIT command; go back to the outer loop // and block for InMemoryMerge. break; } } }
public void run() { // Create a worklist of task attempts to work over. Set<TaskAttemptID> maps = new HashSet<TaskAttemptID>(); for (TaskAttemptID map : localMapFiles.keySet()) { maps.add(map); } while (maps.size() > 0) { try { // If merge is on, block merger.waitForResource(); metrics.threadBusy(); // Copy as much as is possible. doCopy(maps); metrics.threadFree(); } catch (InterruptedException ie) { } catch (Throwable t) { exceptionReporter.reportException(t); } } }
Fetcher<K,V>[] fetchers = new Fetcher[numFetchers]; if (isLocal) { fetchers[0] = new LocalFetcher<K, V>(jobConf, reduceId, scheduler, merger, reporter, metrics, this, reduceTask.getShuffleSecret(), localMapFiles);
public LocalFetcher(JobConf job, TaskAttemptID reduceId, ShuffleSchedulerImpl<K, V> scheduler, MergeManager<K,V> merger, Reporter reporter, ShuffleClientMetrics metrics, ExceptionReporter exceptionReporter, SecretKey shuffleKey, Map<TaskAttemptID, MapOutputFile> localMapFiles) { super(job, reduceId, scheduler, merger, reporter, metrics, exceptionReporter, shuffleKey); this.job = job; this.localMapFiles = localMapFiles; setName("localfetcher#" + id); setDaemon(true); }
/** * The crux of the matter... */ private void doCopy(Set<TaskAttemptID> maps) throws IOException { Iterator<TaskAttemptID> iter = maps.iterator(); while (iter.hasNext()) { TaskAttemptID map = iter.next(); LOG.debug("LocalFetcher " + id + " going to fetch: " + map); if (copyMapOutput(map)) { // Successful copy. Remove this from our worklist. iter.remove(); } else { // We got back a WAIT command; go back to the outer loop // and block for InMemoryMerge. break; } } }