@SuppressWarnings({ "unchecked" }) private void updatePreemptionCounters(TaskAttemptId yarnAttemptID) { if (!countedPreemptions.contains(yarnAttemptID)) { countedPreemptions.add(yarnAttemptID); JobCounterUpdateEvent jce = new JobCounterUpdateEvent(yarnAttemptID .getTaskId().getJobId()); jce.addCounterUpdate(JobCounter.TASKS_REQ_PREEMPT, 1); eventHandler.handle(jce); } }
@Override public void transition(JobImpl job, JobEvent event) { JobCounterUpdateEvent jce = (JobCounterUpdateEvent) event; for (JobCounterUpdateEvent.CounterIncrementalUpdate ci : jce .getCounterUpdates()) { job.jobCounters.findCounter(ci.getCounterKey()).increment( ci.getIncrementValue()); } } }
private static JobCounterUpdateEvent createJobCounterUpdateEventTASucceeded( TaskAttemptImpl taskAttempt) { TaskId taskId = taskAttempt.attemptId.getTaskId(); JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskId.getJobId()); updateMillisCounters(jce, taskAttempt); return jce; }
private static void updateMillisCounters(JobCounterUpdateEvent jce, TaskAttemptImpl taskAttempt) { TaskType taskType = taskAttempt.getID().getTaskId().getTaskType(); long duration = (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime()); int mbRequired = taskAttempt.getMemoryRequired(taskAttempt.conf, taskType); int vcoresRequired = taskAttempt.getCpuRequired(taskAttempt.conf, taskType); int minSlotMemSize = taskAttempt.conf.getInt( YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); int simSlotsRequired = minSlotMemSize == 0 ? 0 : (int) Math.ceil((float) mbRequired / minSlotMemSize); if (taskType == TaskType.MAP) { jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, simSlotsRequired * duration); jce.addCounterUpdate(JobCounter.MB_MILLIS_MAPS, duration * mbRequired); jce.addCounterUpdate(JobCounter.VCORES_MILLIS_MAPS, duration * vcoresRequired); jce.addCounterUpdate(JobCounter.MILLIS_MAPS, duration); } else { jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, simSlotsRequired * duration); jce.addCounterUpdate(JobCounter.MB_MILLIS_REDUCES, duration * mbRequired); jce.addCounterUpdate(JobCounter.VCORES_MILLIS_REDUCES, duration * vcoresRequired); jce.addCounterUpdate(JobCounter.MILLIS_REDUCES, duration); } }
@Override public void transition(JobImpl job, JobEvent event) { JobCounterUpdateEvent jce = (JobCounterUpdateEvent) event; for (JobCounterUpdateEvent.CounterIncrementalUpdate ci : jce .getCounterUpdates()) { job.jobCounters.findCounter(ci.getCounterKey()).increment( ci.getIncrementValue()); } } }
private static JobCounterUpdateEvent createJobCounterUpdateEventTASucceeded( TaskAttemptImpl taskAttempt) { TaskId taskId = taskAttempt.attemptId.getTaskId(); JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskId.getJobId()); updateMillisCounters(jce, taskAttempt); return jce; }
private static void updateMillisCounters(JobCounterUpdateEvent jce, TaskAttemptImpl taskAttempt) { TaskType taskType = taskAttempt.getID().getTaskId().getTaskType(); long duration = (taskAttempt.getFinishTime() - taskAttempt.getLaunchTime()); int mbRequired = taskAttempt.getMemoryRequired(taskAttempt.conf, taskType); int vcoresRequired = taskAttempt.getCpuRequired(taskAttempt.conf, taskType); int minSlotMemSize = taskAttempt.conf.getInt( YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB); int simSlotsRequired = minSlotMemSize == 0 ? 0 : (int) Math.ceil((float) mbRequired / minSlotMemSize); if (taskType == TaskType.MAP) { jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, simSlotsRequired * duration); jce.addCounterUpdate(JobCounter.MB_MILLIS_MAPS, duration * mbRequired); jce.addCounterUpdate(JobCounter.VCORES_MILLIS_MAPS, duration * vcoresRequired); jce.addCounterUpdate(JobCounter.MILLIS_MAPS, duration); } else { jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, simSlotsRequired * duration); jce.addCounterUpdate(JobCounter.MB_MILLIS_REDUCES, duration * mbRequired); jce.addCounterUpdate(JobCounter.VCORES_MILLIS_REDUCES, duration * vcoresRequired); jce.addCounterUpdate(JobCounter.MILLIS_REDUCES, duration); } }
@SuppressWarnings({ "unchecked" }) private void updateCheckpointCounters(TaskId taskId, TaskCheckpointID cid) { JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskId.getJobId()); jce.addCounterUpdate(JobCounter.CHECKPOINTS, 1); eventHandler.handle(jce); jce = new JobCounterUpdateEvent(taskId.getJobId()); jce.addCounterUpdate(JobCounter.CHECKPOINT_BYTES, cid.getCheckpointBytes()); eventHandler.handle(jce); jce = new JobCounterUpdateEvent(taskId.getJobId()); jce.addCounterUpdate(JobCounter.CHECKPOINT_TIME, cid.getCheckpointTime()); eventHandler.handle(jce); }
@Override public void transition(JobImpl job, JobEvent event) { JobCounterUpdateEvent jce = (JobCounterUpdateEvent) event; for (JobCounterUpdateEvent.CounterIncrementalUpdate ci : jce .getCounterUpdates()) { job.jobCounters.findCounter(ci.getCounterKey()).increment( ci.getIncrementValue()); } } }
private static JobCounterUpdateEvent createJobCounterUpdateEventTASucceeded( TaskAttemptImpl taskAttempt) { TaskId taskId = taskAttempt.attemptId.getTaskId(); JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskId.getJobId()); updateMillisCounters(jce, taskAttempt); return jce; }
jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_MAPS, simSlotsAllocated * duration); jce.addCounterUpdate(JobCounter.MB_MILLIS_MAPS, duration * mbAllocated); jce.addCounterUpdate(JobCounter.VCORES_MILLIS_MAPS, duration * vcoresAllocated); jce.addCounterUpdate(JobCounter.MILLIS_MAPS, duration); } else { jce.addCounterUpdate(JobCounter.SLOTS_MILLIS_REDUCES, simSlotsAllocated * duration); jce.addCounterUpdate(JobCounter.MB_MILLIS_REDUCES, duration * mbAllocated); jce.addCounterUpdate(JobCounter.VCORES_MILLIS_REDUCES, duration * vcoresAllocated); jce.addCounterUpdate(JobCounter.MILLIS_REDUCES, duration);
@SuppressWarnings("unchecked") private ContainerRequest assignToFailedMap(Container allocated) { //try to assign to earlierFailedMaps if present ContainerRequest assigned = null; while (assigned == null && earlierFailedMaps.size() > 0 && canAssignMaps()) { TaskAttemptId tId = earlierFailedMaps.removeFirst(); if (maps.containsKey(tId)) { assigned = maps.remove(tId); JobCounterUpdateEvent jce = new JobCounterUpdateEvent(assigned.attemptID.getTaskId().getJobId()); jce.addCounterUpdate(JobCounter.OTHER_LOCAL_MAPS, 1); eventHandler.handle(jce); LOG.info("Assigned from earlierFailedMaps"); break; } } return assigned; }
} else if (current instanceof JobCounterUpdateEvent) { JobCounterUpdateEvent jcue = (JobCounterUpdateEvent) current; boolean containsUpdates = jcue.getCounterUpdates().size() > 0; + jcue.getCounterUpdates().get(0).getCounterKey() + " " + jcue.getCounterUpdates().get(0).getIncrementValue()); if (jcue.getCounterUpdates().get(0).getCounterKey() == JobCounter.NUM_FAILED_MAPS) { totalFailedMaps += jcue.getCounterUpdates().get(0) .getIncrementValue(); } else if (jcue.getCounterUpdates().get(0).getCounterKey() == JobCounter.TOTAL_LAUNCHED_MAPS) { totalLaunchedMaps += jcue.getCounterUpdates().get(0) .getIncrementValue();
@SuppressWarnings("unchecked") private ContainerRequest assignToFailedMap(Container allocated) { //try to assign to earlierFailedMaps if present ContainerRequest assigned = null; while (assigned == null && earlierFailedMaps.size() > 0 && canAssignMaps()) { TaskAttemptId tId = earlierFailedMaps.removeFirst(); if (maps.containsKey(tId)) { assigned = maps.remove(tId); JobCounterUpdateEvent jce = new JobCounterUpdateEvent(assigned.attemptID.getTaskId().getJobId()); jce.addCounterUpdate(JobCounter.OTHER_LOCAL_MAPS, 1); eventHandler.handle(jce); LOG.info("Assigned from earlierFailedMaps"); break; } } return assigned; }
@SuppressWarnings("unchecked") private ContainerRequest assignToFailedMap(Container allocated) { //try to assign to earlierFailedMaps if present ContainerRequest assigned = null; while (assigned == null && earlierFailedMaps.size() > 0 && canAssignMaps()) { TaskAttemptId tId = earlierFailedMaps.removeFirst(); if (maps.containsKey(tId)) { assigned = maps.remove(tId); JobCounterUpdateEvent jce = new JobCounterUpdateEvent(assigned.attemptID.getTaskId().getJobId()); jce.addCounterUpdate(JobCounter.OTHER_LOCAL_MAPS, 1); eventHandler.handle(jce); LOG.info("Assigned from earlierFailedMaps"); break; } } return assigned; }
private static JobCounterUpdateEvent createJobCounterUpdateEventTAFailed( TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted) { TaskType taskType = taskAttempt.getID().getTaskId().getTaskType(); JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId()); if (taskType == TaskType.MAP) { jce.addCounterUpdate(JobCounter.NUM_FAILED_MAPS, 1); } else { jce.addCounterUpdate(JobCounter.NUM_FAILED_REDUCES, 1); } if (!taskAlreadyCompleted) { updateMillisCounters(jce, taskAttempt); } return jce; }
private static JobCounterUpdateEvent createJobCounterUpdateEventTAFailed( TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted) { TaskType taskType = taskAttempt.getID().getTaskId().getTaskType(); JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId()); if (taskType == TaskType.MAP) { jce.addCounterUpdate(JobCounter.NUM_FAILED_MAPS, 1); } else { jce.addCounterUpdate(JobCounter.NUM_FAILED_REDUCES, 1); } if (!taskAlreadyCompleted) { updateMillisCounters(jce, taskAttempt); } return jce; }
private static JobCounterUpdateEvent createJobCounterUpdateEventTAKilled( TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted) { TaskType taskType = taskAttempt.getID().getTaskId().getTaskType(); JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId()); if (taskType == TaskType.MAP) { jce.addCounterUpdate(JobCounter.NUM_KILLED_MAPS, 1); } else { jce.addCounterUpdate(JobCounter.NUM_KILLED_REDUCES, 1); } if (!taskAlreadyCompleted) { updateMillisCounters(jce, taskAttempt); } return jce; }
private static JobCounterUpdateEvent createJobCounterUpdateEventTAFailed( TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted) { TaskType taskType = taskAttempt.getID().getTaskId().getTaskType(); JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId()); if (taskType == TaskType.MAP) { jce.addCounterUpdate(JobCounter.NUM_FAILED_MAPS, 1); } else { jce.addCounterUpdate(JobCounter.NUM_FAILED_REDUCES, 1); } if (!taskAlreadyCompleted) { updateMillisCounters(jce, taskAttempt); } return jce; }
private static JobCounterUpdateEvent createJobCounterUpdateEventTAKilled( TaskAttemptImpl taskAttempt, boolean taskAlreadyCompleted) { TaskType taskType = taskAttempt.getID().getTaskId().getTaskType(); JobCounterUpdateEvent jce = new JobCounterUpdateEvent(taskAttempt.getID().getTaskId().getJobId()); if (taskType == TaskType.MAP) { jce.addCounterUpdate(JobCounter.NUM_KILLED_MAPS, 1); } else { jce.addCounterUpdate(JobCounter.NUM_KILLED_REDUCES, 1); } if (!taskAlreadyCompleted) { updateMillisCounters(jce, taskAttempt); } return jce; }