this.umbilical = umbilical; this.reporter = reporter; this.metrics = new ShuffleClientMetrics(reduceId, jobConf); this.copyPhase = copyPhase; this.taskStatus = status;
metrics.inputBytes(n); reporter.progress();
public void run() { // Create a worklist of task attempts to work over. Set<TaskAttemptID> maps = new HashSet<TaskAttemptID>(); for (TaskAttemptID map : localMapFiles.keySet()) { maps.add(map); } while (maps.size() > 0) { try { // If merge is on, block merger.waitForResource(); metrics.threadBusy(); // Copy as much as is possible. doCopy(maps); metrics.threadFree(); } catch (InterruptedException ie) { } catch (Throwable t) { exceptionReporter.reportException(t); } } }
metrics.successFetch(); return true; } catch (IOException ioe) { metrics.failedFetch(); return false;
metrics.successFetch(); return null; } catch (IOException ioe) { metrics.failedFetch(); return new TaskAttemptID[] {mapId};
public void run() { // Create a worklist of task attempts to work over. Set<TaskAttemptID> maps = new HashSet<TaskAttemptID>(); for (TaskAttemptID map : localMapFiles.keySet()) { maps.add(map); } while (maps.size() > 0) { try { // If merge is on, block merger.waitForResource(); metrics.threadBusy(); // Copy as much as is possible. doCopy(maps); metrics.threadFree(); } catch (InterruptedException ie) { } catch (Throwable t) { exceptionReporter.reportException(t); } } }
metrics.successFetch(); return null; } catch (IOException ioe) { metrics.failedFetch(); return new TaskAttemptID[] {mapId};
metrics.inputBytes(n); reporter.progress();
@Override public void init(ShuffleConsumerPlugin.Context context) { this.context = context; this.reduceId = context.getReduceId(); this.jobConf = context.getJobConf(); this.umbilical = context.getUmbilical(); this.reporter = context.getReporter(); this.metrics = new ShuffleClientMetrics(reduceId, jobConf); this.copyPhase = context.getCopyPhase(); this.taskStatus = context.getStatus(); this.reduceTask = context.getReduceTask(); this.localMapFiles = context.getLocalMapFiles(); scheduler = new ShuffleSchedulerImpl<K, V>(jobConf, taskStatus, reduceId, this, copyPhase, context.getShuffledMapsCounter(), context.getReduceShuffleBytes(), context.getFailedShuffleCounter()); merger = createMergeManager(context); }
public void run() { // Create a worklist of task attempts to work over. Set<TaskAttemptID> maps = new HashSet<TaskAttemptID>(); for (TaskAttemptID map : localMapFiles.keySet()) { maps.add(map); } while (maps.size() > 0) { try { // If merge is on, block merger.waitForResource(); metrics.threadBusy(); // Copy as much as is possible. doCopy(maps); metrics.threadFree(); } catch (InterruptedException ie) { } catch (Throwable t) { exceptionReporter.reportException(t); } } }
metrics.successFetch(); return null; } catch (IOException ioe) { metrics.failedFetch(); return new TaskAttemptID[] {mapId};
metrics.inputBytes(n); reporter.progress();
@Override public void init(ShuffleConsumerPlugin.Context context) { this.context = context; this.reduceId = context.getReduceId(); this.jobConf = context.getJobConf(); this.umbilical = context.getUmbilical(); this.reporter = context.getReporter(); this.metrics = new ShuffleClientMetrics(reduceId, jobConf); this.copyPhase = context.getCopyPhase(); this.taskStatus = context.getStatus(); this.reduceTask = context.getReduceTask(); this.localMapFiles = context.getLocalMapFiles(); scheduler = new ShuffleSchedulerImpl<K, V>(jobConf, taskStatus, reduceId, this, copyPhase, context.getShuffledMapsCounter(), context.getReduceShuffleBytes(), context.getFailedShuffleCounter()); merger = createMergeManager(context); }
public void run() { // Create a worklist of task attempts to work over. Set<TaskAttemptID> maps = new HashSet<TaskAttemptID>(); for (TaskAttemptID map : localMapFiles.keySet()) { maps.add(map); } while (maps.size() > 0) { try { // If merge is on, block merger.waitForResource(); metrics.threadBusy(); // Copy as much as is possible. doCopy(maps); metrics.threadFree(); } catch (InterruptedException ie) { } catch (Throwable t) { exceptionReporter.reportException(t); } } }
metrics.successFetch(); return null; } catch (IOException ioe) { metrics.failedFetch(); return new TaskAttemptID[] {mapId};
metrics.inputBytes(n); reporter.progress();
@Override public void init(ShuffleConsumerPlugin.Context context) { this.context = context; this.reduceId = context.getReduceId(); this.jobConf = context.getJobConf(); this.umbilical = context.getUmbilical(); this.reporter = context.getReporter(); this.metrics = new ShuffleClientMetrics(reduceId, jobConf); this.copyPhase = context.getCopyPhase(); this.taskStatus = context.getStatus(); this.reduceTask = context.getReduceTask(); this.localMapFiles = context.getLocalMapFiles(); scheduler = new ShuffleSchedulerImpl<K, V>(jobConf, taskStatus, reduceId, this, copyPhase, context.getShuffledMapsCounter(), context.getReduceShuffleBytes(), context.getFailedShuffleCounter()); merger = createMergeManager(context); }
public void run() { try { while (!stopped && !Thread.currentThread().isInterrupted()) { MapHost host = null; try { // If merge is on, block merger.waitForResource(); // Get a host to shuffle from host = scheduler.getHost(); metrics.threadBusy(); // Shuffle copyFromHost(host); } finally { if (host != null) { scheduler.freeHost(host); metrics.threadFree(); } } } } catch (InterruptedException ie) { return; } catch (Throwable t) { exceptionReporter.reportException(t); } }
metrics.inputBytes(memory.length); reporter.progress(); LOG.info("Read " + memory.length + " bytes from map-output for " +
@Override public void init(ShuffleConsumerPlugin.Context context) { this.context = context; this.reduceId = context.getReduceId(); this.jobConf = context.getJobConf(); this.umbilical = context.getUmbilical(); this.reporter = context.getReporter(); this.metrics = new ShuffleClientMetrics(reduceId, jobConf); this.copyPhase = context.getCopyPhase(); this.taskStatus = context.getStatus(); this.reduceTask = context.getReduceTask(); this.localMapFiles = context.getLocalMapFiles(); scheduler = new ShuffleSchedulerImpl<K, V>(jobConf, taskStatus, reduceId, this, copyPhase, context.getShuffledMapsCounter(), context.getReduceShuffleBytes(), context.getFailedShuffleCounter()); merger = createMergeManager(context); }