currTaskContext.getTaskAttemptID(), currTaskContext.getProgressible());
private Counter getTopicSkipOldCounter() { try { //In Hadoop 2, TaskAttemptContext.getCounter() is available Method getCounterMethod = context.getClass().getMethod("getCounter", String.class, String.class); return ((Counter) getCounterMethod.invoke(context, "total", "skip-old")); } catch (NoSuchMethodException e) { //In Hadoop 1, TaskAttemptContext.getCounter() is not available //Have to cast context to TaskAttemptContext in the mapred package, then get a StatusReporter instance org.apache.hadoop.mapred.TaskAttemptContext mapredContext = (org.apache.hadoop.mapred.TaskAttemptContext) context; return ((StatusReporter) mapredContext.getProgressible()).getCounter("total", "skip-old"); } catch (IllegalArgumentException e) { log.error("IllegalArgumentException while obtaining counter 'total:skip-old': " + e.getMessage()); throw new RuntimeException(e); } catch (IllegalAccessException e) { log.error("IllegalAccessException while obtaining counter 'total:skip-old': " + e.getMessage()); throw new RuntimeException(e); } catch (InvocationTargetException e) { log.error("InvocationTargetException obtaining counter 'total:skip-old': " + e.getMessage()); throw new RuntimeException(e); } }
private long getRecordCountCounter(TaskAttemptContext context) { try { //In Hadoop 2, TaskAttemptContext.getCounter() is available Method getCounterMethod = context.getClass().getMethod("getCounter", Enum.class); return ((Counter) getCounterMethod.invoke(context, AvroKeyReducer.EVENT_COUNTER.RECORD_COUNT)).getValue(); } catch (NoSuchMethodException e) { //In Hadoop 1, TaskAttemptContext.getCounter() is not available //Have to cast context to TaskAttemptContext in the mapred package, then get a StatusReporter instance org.apache.hadoop.mapred.TaskAttemptContext mapredContext = (org.apache.hadoop.mapred.TaskAttemptContext) context; return ((StatusReporter) mapredContext.getProgressible()).getCounter(AvroKeyReducer.EVENT_COUNTER.RECORD_COUNT) .getValue(); } catch (Exception e) { throw new RuntimeException("Error reading record count counter", e); } }
private Counter getTopicSkipOldCounter() { try { //In Hadoop 2, TaskAttemptContext.getCounter() is available Method getCounterMethod = context.getClass().getMethod("getCounter", String.class, String.class); return ((Counter) getCounterMethod.invoke(context, "total", "skip-old")); } catch (NoSuchMethodException e) { //In Hadoop 1, TaskAttemptContext.getCounter() is not available //Have to cast context to TaskAttemptContext in the mapred package, then get a StatusReporter instance org.apache.hadoop.mapred.TaskAttemptContext mapredContext = (org.apache.hadoop.mapred.TaskAttemptContext) context; return ((StatusReporter) mapredContext.getProgressible()).getCounter("total", "skip-old"); } catch (IllegalArgumentException e) { log.error("IllegalArgumentException while obtaining counter 'total:skip-old': " + e.getMessage()); throw new RuntimeException(e); } catch (IllegalAccessException e) { log.error("IllegalAccessException while obtaining counter 'total:skip-old': " + e.getMessage()); throw new RuntimeException(e); } catch (InvocationTargetException e) { log.error("InvocationTargetException obtaining counter 'total:skip-old': " + e.getMessage()); throw new RuntimeException(e); } }
public void abortTask(TaskAttemptContext context) throws IOException { Path taskOutputPath = getTempTaskOutputPath(context); try { if (taskOutputPath != null) { FileSystem fs = taskOutputPath.getFileSystem(context.getJobConf()); context.getProgressible().progress(); if (!fs.delete(taskOutputPath, true)) { LOG.warn("Deleting output in " + taskOutputPath + " returns false"); } } } catch (IOException ie) { LOG.warn("Error discarding output in " + taskOutputPath, ie); throw ie; } }
public void abortTask(TaskAttemptContext context) throws IOException { Path taskOutputPath = getTempTaskOutputPath(context); if (taskOutputPath != null) { FileSystem fs = taskOutputPath.getFileSystem(context.getJobConf()); context.getProgressible().progress(); fs.delete(taskOutputPath, true); } }
public void abortTask(TaskAttemptContext context) { Path taskOutputPath = getTempTaskOutputPath(context); try { if (taskOutputPath != null) { FileSystem fs = taskOutputPath.getFileSystem(context.getJobConf()); context.getProgressible().progress(); fs.delete(taskOutputPath, true); } } catch (IOException ie) { LOG.warn("Error discarding output" + StringUtils.stringifyException(ie)); } }
public boolean needsTaskCommit(TaskAttemptContext context) throws IOException { Path taskOutputPath = getTempTaskOutputPath(context); if (taskOutputPath != null) { context.getProgressible().progress(); // Get the file-system for the task output directory FileSystem fs = taskOutputPath.getFileSystem(context.getJobConf()); // since task output path is created on demand, // if it exists, task needs a commit if (fs.exists(taskOutputPath)) { return true; } } return false; }
public boolean needsTaskCommit(TaskAttemptContext context) throws IOException { Path taskOutputPath = getTempTaskOutputPath(context); try { if (taskOutputPath != null) { context.getProgressible().progress(); // Get the file-system for the task output directory FileSystem fs = taskOutputPath.getFileSystem(context.getJobConf()); // since task output path is created on demand, // if it exists, task needs a commit if (fs.exists(taskOutputPath)) { return true; } } } catch (IOException ioe) { LOG.warn("Error when checking " + taskOutputPath, ioe); throw ioe; } return false; }
public void commitTask(TaskAttemptContext context) throws IOException { Path taskOutputPath = getTempTaskOutputPath(context); TaskAttemptID attemptId = context.getTaskAttemptID(); JobConf job = context.getJobConf(); if (taskOutputPath != null) { FileSystem fs = taskOutputPath.getFileSystem(job); context.getProgressible().progress(); if (fs.exists(taskOutputPath)) { Path jobOutputPath = taskOutputPath.getParent().getParent(); // Move the task outputs to their final place moveTaskOutputs(context, fs, jobOutputPath, taskOutputPath); // Delete the temporary task-specific output directory if (!fs.delete(taskOutputPath, true)) { LOG.info("Failed to delete the temporary output" + " directory of task: " + attemptId + " - " + taskOutputPath); } LOG.info("Saved output of task '" + attemptId + "' to " + jobOutputPath); } } }
public boolean needsTaskCommit(TaskAttemptContext context) throws IOException { try { Path taskOutputPath = getTempTaskOutputPath(context); if (taskOutputPath != null) { context.getProgressible().progress(); // Get the file-system for the task output directory FileSystem fs = taskOutputPath.getFileSystem(context.getJobConf()); // since task output path is created on demand, // if it exists, task needs a commit if (fs.exists(taskOutputPath)) { return true; } } } catch (IOException ioe) { throw ioe; } return false; }
public void commitTask(TaskAttemptContext context) throws IOException { Path taskOutputPath = getTempTaskOutputPath(context); TaskAttemptID attemptId = context.getTaskAttemptID(); JobConf job = context.getJobConf(); if (taskOutputPath != null) { FileSystem fs = taskOutputPath.getFileSystem(job); context.getProgressible().progress(); if (fs.exists(taskOutputPath)) { Path jobOutputPath = taskOutputPath.getParent().getParent(); // Move the task outputs to their final place moveTaskOutputs(context, fs, jobOutputPath, taskOutputPath); // Delete the temporary task-specific output directory if (!fs.delete(taskOutputPath, true)) { LOG.info("Failed to delete the temporary output" + " directory of task: " + attemptId + " - " + taskOutputPath); } LOG.info("Saved output of task '" + attemptId + "' to " + jobOutputPath); } } }
public void commitTask(TaskAttemptContext context) throws IOException { Path taskOutputPath = getTempTaskOutputPath(context); TaskAttemptID attemptId = context.getTaskAttemptID(); JobConf job = context.getJobConf(); if (taskOutputPath != null) { FileSystem fs = taskOutputPath.getFileSystem(job); context.getProgressible().progress(); if (fs.exists(taskOutputPath)) { Path jobOutputPath = taskOutputPath.getParent().getParent(); // Move the task outputs to their final place moveTaskOutputs(context, fs, jobOutputPath, taskOutputPath); // Delete the temporary task-specific output directory if (!fs.delete(taskOutputPath, true)) { LOG.info("Failed to delete the temporary output" + " directory of task: " + attemptId + " - " + taskOutputPath); } LOG.info("Saved output of task '" + attemptId + "' to " + jobOutputPath); } } }
throws IOException { TaskAttemptID attemptId = context.getTaskAttemptID(); context.getProgressible().progress(); if (fs.isFile(taskOutput)) { Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput,
throws IOException { TaskAttemptID attemptId = context.getTaskAttemptID(); context.getProgressible().progress(); if (fs.isFile(taskOutput)) { Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput,
throws IOException { TaskAttemptID attemptId = context.getTaskAttemptID(); context.getProgressible().progress(); if (fs.isFile(taskOutput)) { Path finalOutputPath = getFinalPath(jobOutputDir, taskOutput,
currTaskContext.getTaskAttemptID(), currTaskContext.getProgressible());
currTaskContext.getTaskAttemptID(), currTaskContext.getProgressible());
currTaskContext.getTaskAttemptID(), currTaskContext.getProgressible());
currTaskContext.getTaskAttemptID(), currTaskContext.getProgressible());