private synchronized void waitForValidCommitWindow() throws InterruptedException { long lastHeartbeatTime = rmHeartbeatHandler.getLastHeartbeatTime(); long now = context.getClock().getTime(); while (now - lastHeartbeatTime > commitWindowMs) { rmHeartbeatHandler.runOnNextHeartbeat(new Runnable() { @Override public void run() { synchronized (EventProcessor.this) { EventProcessor.this.notify(); } } }); wait(); lastHeartbeatTime = rmHeartbeatHandler.getLastHeartbeatTime(); now = context.getClock().getTime(); } } }
when(rmHeartbeatHandler.getLastHeartbeatTime()) .thenReturn(clock.getTime()); canCommit = listener.canCommit(tid);
/** * Child checking whether it can commit. * * <br> * Commit is a two-phased protocol. First the attempt informs the * ApplicationMaster that it is * {@link #commitPending(TaskAttemptID, TaskStatus)}. Then it repeatedly polls * the ApplicationMaster whether it {@link #canCommit(TaskAttemptID)} This is * a legacy from the centralized commit protocol handling by the JobTracker. */ @Override public boolean canCommit(TaskAttemptID taskAttemptID) throws IOException { LOG.info("Commit go/no-go request from " + taskAttemptID.toString()); // An attempt is asking if it can commit its output. This can be decided // only by the task which is managing the multiple attempts. So redirect the // request there. org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID = TypeConverter.toYarn(taskAttemptID); taskHeartbeatHandler.progressing(attemptID); // tell task to retry later if AM has not heard from RM within the commit // window to help avoid double-committing in a split-brain situation long now = context.getClock().getTime(); if (now - rmHeartbeatHandler.getLastHeartbeatTime() > commitWindowMs) { return false; } Job job = context.getJob(attemptID.getTaskId().getJobId()); Task task = job.getTask(attemptID.getTaskId()); return task.canCommit(attemptID); }
private synchronized void waitForValidCommitWindow() throws InterruptedException { long lastHeartbeatTime = rmHeartbeatHandler.getLastHeartbeatTime(); long now = context.getClock().getTime(); while (now - lastHeartbeatTime > commitWindowMs) { rmHeartbeatHandler.runOnNextHeartbeat(new Runnable() { @Override public void run() { synchronized (EventProcessor.this) { EventProcessor.this.notify(); } } }); wait(); lastHeartbeatTime = rmHeartbeatHandler.getLastHeartbeatTime(); now = context.getClock().getTime(); } } }
/** * Child checking whether it can commit. * * <br> * Commit is a two-phased protocol. First the attempt informs the * ApplicationMaster that it is * {@link #commitPending(TaskAttemptID, TaskStatus)}. Then it repeatedly polls * the ApplicationMaster whether it {@link #canCommit(TaskAttemptID)} This is * a legacy from the centralized commit protocol handling by the JobTracker. */ @Override public boolean canCommit(TaskAttemptID taskAttemptID) throws IOException { LOG.info("Commit go/no-go request from " + taskAttemptID.toString()); // An attempt is asking if it can commit its output. This can be decided // only by the task which is managing the multiple attempts. So redirect the // request there. org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID = TypeConverter.toYarn(taskAttemptID); taskHeartbeatHandler.progressing(attemptID); // tell task to retry later if AM has not heard from RM within the commit // window to help avoid double-committing in a split-brain situation long now = context.getClock().getTime(); if (now - rmHeartbeatHandler.getLastHeartbeatTime() > commitWindowMs) { return false; } Job job = context.getJob(attemptID.getTaskId().getJobId()); Task task = job.getTask(attemptID.getTaskId()); return task.canCommit(attemptID); }
private synchronized void waitForValidCommitWindow() throws InterruptedException { long lastHeartbeatTime = rmHeartbeatHandler.getLastHeartbeatTime(); long now = context.getClock().getTime(); while (now - lastHeartbeatTime > commitWindowMs) { rmHeartbeatHandler.runOnNextHeartbeat(new Runnable() { @Override public void run() { synchronized (EventProcessor.this) { EventProcessor.this.notify(); } } }); wait(); lastHeartbeatTime = rmHeartbeatHandler.getLastHeartbeatTime(); now = context.getClock().getTime(); } } }
/** * Child checking whether it can commit. * * <br> * Commit is a two-phased protocol. First the attempt informs the * ApplicationMaster that it is * {@link #commitPending(TaskAttemptID, TaskStatus)}. Then it repeatedly polls * the ApplicationMaster whether it {@link #canCommit(TaskAttemptID)} This is * a legacy from the centralized commit protocol handling by the JobTracker. */ @Override public boolean canCommit(TaskAttemptID taskAttemptID) throws IOException { LOG.info("Commit go/no-go request from " + taskAttemptID.toString()); // An attempt is asking if it can commit its output. This can be decided // only by the task which is managing the multiple attempts. So redirect the // request there. org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId attemptID = TypeConverter.toYarn(taskAttemptID); taskHeartbeatHandler.progressing(attemptID); // tell task to retry later if AM has not heard from RM within the commit // window to help avoid double-committing in a split-brain situation long now = context.getClock().getTime(); if (now - rmHeartbeatHandler.getLastHeartbeatTime() > commitWindowMs) { return false; } Job job = context.getJob(attemptID.getTaskId().getJobId()); Task task = job.getTask(attemptID.getTaskId()); return task.canCommit(attemptID); }