context.getJobID(), context.getTaskAttemptID(), taskContext.getWorkingDirectory()
@Override public JobID getJobID() { return delegate.getJobID(); }
private TaskAttemptContext getContext(String namedOutput, TaskAttemptContext baseContext) throws IOException { Job job = getJob(baseContext.getJobID(), namedOutput, baseContext.getConfiguration()); configureJob(namedOutput, job, outputs.get(namedOutput)); return getTaskContext(baseContext, job); }
String getJobId() { return taskAttemptContext.getJobID().getJtIdentifier(); }
@Override public void setupTask(TaskAttemptContext taskContext) throws IOException { IOException exception = null; for (OutputCommitter component : components) { try { component.setupTask(taskContext); } catch (IOException e) { LOG.warn(MessageFormat.format( "Failed to setup task (JobID={1}, TaskAttemptID={2}, committer={0})", component.getClass().getName(), taskContext.getJobID(), taskContext.getTaskAttemptID()), e); if (exception == null) { exception = e; } } } if (exception != null) { throw exception; } }
@Override public void abortTask(TaskAttemptContext taskContext) throws IOException { IOException exception = null; for (OutputCommitter component : components) { try { component.abortTask(taskContext); } catch (IOException e) { LOG.warn(MessageFormat.format( "Failed to abort task (JobID={1}, TaskAttemptID={2}, committer={0})", component.getClass().getName(), taskContext.getJobID(), taskContext.getTaskAttemptID()), e); if (exception == null) { exception = e; } } } if (exception != null) { throw exception; } } }
public static OutputCommitter getOutputCommitter(TaskAttemptContext tac) throws IOException, InterruptedException { Map<String, OutputConfig> outputs = getNamedOutputs(tac.getConfiguration()); Map<String, OutputCommitter> committers = Maps.newHashMap(); for (Map.Entry<String, OutputConfig> e : outputs.entrySet()) { String namedOutput = e.getKey(); Job job = getJob(tac.getJobID(), e.getKey(), tac.getConfiguration()); OutputFormat fmt = getOutputFormat(namedOutput, job, e.getValue()); TaskAttemptContext taskContext = getTaskContext(tac, job); OutputCommitter oc = fmt.getOutputCommitter(taskContext); committers.put(namedOutput, oc); } return new CompositeOutputCommitter(outputs, committers); }
@Override public void commitTask(TaskAttemptContext taskContext) throws IOException { IOException exception = null; for (OutputCommitter component : components) { try { if (component.needsTaskCommit(taskContext)) { component.commitTask(taskContext); } } catch (IOException e) { LOG.warn(MessageFormat.format( "Failed to commit task (JobID={1}, TaskAttemptID={2}, committer={0})", component.getClass().getName(), taskContext.getJobID(), taskContext.getTaskAttemptID()), e); if (exception == null) { exception = e; } } } if (exception != null) { throw exception; } }
private static OutputCommitter initOutputCommitter( OutputFormat<?, ?> outputFormatObj, Configuration conf, TaskAttemptContext taskAttemptContext) throws IllegalStateException { OutputCommitter outputCommitter; try { outputCommitter = outputFormatObj.getOutputCommitter(taskAttemptContext); if (outputCommitter != null) { outputCommitter.setupJob(new JobContextImpl(conf, taskAttemptContext.getJobID())); } } catch (Exception e) { throw new IllegalStateException("Unable to create OutputCommitter object: ", e); } return outputCommitter; }
private RecordWriter<KeyT, ValueT> initRecordWriter( OutputFormat<KeyT, ValueT> outputFormatObj, TaskAttemptContext taskAttemptContext) throws IllegalStateException { try { LOGGER.info( "Creating new RecordWriter for task {} of Job with id {}.", taskAttemptContext.getTaskAttemptID().getTaskID().getId(), taskAttemptContext.getJobID().getJtIdentifier()); return outputFormatObj.getRecordWriter(taskAttemptContext); } catch (InterruptedException | IOException e) { throw new IllegalStateException("Unable to create RecordWriter object: ", e); } }
@Before public void setup() throws Exception { Configuration conf = new Configuration(); input = ClassLoader.getSystemClassLoader().getResource("test.cram").getFile(); reference = ClassLoader.getSystemClassLoader().getResource("auxf.fa").toURI().toString(); conf.set("mapred.input.dir", "file://" + input); conf.set(CRAMInputFormat.REFERENCE_SOURCE_PATH_PROPERTY, reference); taskAttemptContext = new TaskAttemptContextImpl(conf, mock(TaskAttemptID.class)); jobContext = new JobContextImpl(conf, taskAttemptContext.getJobID()); }
@Before public void setup() throws Exception { Configuration conf = new Configuration(); input = ClassLoader.getSystemClassLoader().getResource("mini-chr1-chr2.fasta").getFile(); conf.set("mapred.input.dir", "file://" + input); // Input fasta is 600 bytes, so this gets us 3 FileInputFormat splits. conf.set(FileInputFormat.SPLIT_MAXSIZE, "200"); taskAttemptContext = new TaskAttemptContextImpl(conf, mock(TaskAttemptID.class)); jobContext = new JobContextImpl(conf, taskAttemptContext.getJobID()); }
@Before public void setup() throws Exception { Configuration conf = new Configuration(); input = ClassLoader.getSystemClassLoader().getResource("test.cram").getFile(); reference = ClassLoader.getSystemClassLoader().getResource("auxf.fa").toURI().toString(); String referenceIndex = ClassLoader.getSystemClassLoader().getResource("auxf.fa.fai") .toURI().toString(); conf.set("mapred.input.dir", "file://" + input); URI hdfsRef = clusterUri.resolve("/tmp/auxf.fa"); URI hdfsRefIndex = clusterUri.resolve("/tmp/auxf.fa.fai"); Files.copy(Paths.get(URI.create(reference)), Paths.get(hdfsRef)); Files.copy(Paths.get(URI.create(referenceIndex)), Paths.get(hdfsRefIndex)); conf.set(CRAMInputFormat.REFERENCE_SOURCE_PATH_PROPERTY, hdfsRef.toString()); taskAttemptContext = new TaskAttemptContextImpl(conf, mock(TaskAttemptID.class)); jobContext = new JobContextImpl(conf, taskAttemptContext.getJobID()); }
@Before public void setup() throws Exception { Configuration conf = new Configuration(); input = ClassLoader.getSystemClassLoader().getResource("test.sam").getFile(); conf.set("mapred.input.dir", "file://" + input); taskAttemptContext = new TaskAttemptContextImpl(conf, mock(TaskAttemptID.class)); jobContext = new JobContextImpl(conf, taskAttemptContext.getJobID()); }
private void completeSetup(boolean boundedTraversal, List<Interval> intervals, boolean traverseUnplacedUnmapped) { Configuration conf = new Configuration(); conf.set("mapred.input.dir", "file://" + input); if (boundedTraversal) { BAMInputFormat.setTraversalParameters(conf, intervals, traverseUnplacedUnmapped); } taskAttemptContext = new TaskAttemptContextImpl(conf, mock(TaskAttemptID.class)); jobContext = new JobContextImpl(conf, taskAttemptContext.getJobID()); }
@Test(expectedExceptions = IOException.class) public void testTableDoesntExist() throws Exception { Configuration conf = new Configuration(); HiveOutputDescription outputDesc = new HiveOutputDescription(); outputDesc.getTableDesc().setTableName("doesnt-exist"); OutputConf outputConf = new OutputConf(conf, PROFILE_ID); outputConf.writeOutputDescription(outputDesc); HiveApiOutputFormat outputFormat = new HiveApiOutputFormat(); outputFormat.setMyProfileId(PROFILE_ID); JobConf jobConf = new JobConf(conf); TaskAttemptContext taskContext = new HackTaskAttemptContext(jobConf, new TaskAttemptID()); JobContext jobContext = new HackJobContext(jobConf, taskContext.getJobID()); outputFormat.checkOutputSpecs(jobContext); fail(); } }
@Test(expectedExceptions = IOException.class) public void testTableDoesntExist() throws Exception { Configuration conf = new Configuration(); HiveOutputDescription outputDesc = new HiveOutputDescription(); outputDesc.getTableDesc().setTableName("doesnt-exist"); OutputConf outputConf = new OutputConf(conf, PROFILE_ID); outputConf.writeOutputDescription(outputDesc); HiveApiOutputFormat outputFormat = new HiveApiOutputFormat(); outputFormat.setMyProfileId(PROFILE_ID); JobConf jobConf = new JobConf(conf); TaskAttemptContext taskContext = new HackTaskAttemptContext(jobConf, new TaskAttemptID()); JobContext jobContext = new HackJobContext(jobConf, taskContext.getJobID()); outputFormat.checkOutputSpecs(jobContext); fail(); } }
@Test(expectedExceptions = IOException.class) public void testExceptionAfterTooManyRetriesWhenTableDoesNotExist() throws Exception { HiveMetastores.setTestClient(new FaultyThriftHiveMetastore( BackoffRetryTask.NUM_TRIES.getDefaultValue())); Configuration conf = new Configuration(); conf.setLong(BackoffRetryTask.INITIAL_RETRY_DELAY_MSEC.getKey(), 100); HiveOutputDescription outputDesc = new HiveOutputDescription(); outputDesc.getTableDesc().setTableName("doesnt-exist"); OutputConf outputConf = new OutputConf(conf, PROFILE_ID); outputConf.writeOutputDescription(outputDesc); HiveApiOutputFormat outputFormat = new HiveApiOutputFormat(); outputFormat.setMyProfileId(PROFILE_ID); JobConf jobConf = new JobConf(conf); TaskAttemptContext taskContext = new HackTaskAttemptContext(jobConf, new TaskAttemptID()); JobContext jobContext = new HackJobContext(jobConf, taskContext.getJobID()); outputFormat.checkOutputSpecs(jobContext); fail(); }
@Test(expectedExceptions = IOException.class) public void testExceptionAfterTooManyRetriesWhenTableDoesNotExist() throws Exception { HiveMetastores.setTestClient(new FaultyThriftHiveMetastore( BackoffRetryTask.NUM_TRIES.getDefaultValue())); Configuration conf = new Configuration(); conf.setLong(BackoffRetryTask.INITIAL_RETRY_DELAY_MSEC.getKey(), 100); HiveOutputDescription outputDesc = new HiveOutputDescription(); outputDesc.getTableDesc().setTableName("doesnt-exist"); OutputConf outputConf = new OutputConf(conf, PROFILE_ID); outputConf.writeOutputDescription(outputDesc); HiveApiOutputFormat outputFormat = new HiveApiOutputFormat(); outputFormat.setMyProfileId(PROFILE_ID); JobConf jobConf = new JobConf(conf); TaskAttemptContext taskContext = new HackTaskAttemptContext(jobConf, new TaskAttemptID()); JobContext jobContext = new HackJobContext(jobConf, taskContext.getJobID()); outputFormat.checkOutputSpecs(jobContext); fail(); }
@Test public void testRecoveredFromFailuresAfterRetries() throws Exception { FaultyThriftHiveMetastore metastore = new FaultyThriftHiveMetastore( BackoffRetryTask.NUM_TRIES.getDefaultValue() - 1); Configuration conf = new Configuration(); conf.setLong(BackoffRetryTask.INITIAL_RETRY_DELAY_MSEC.getKey(), 100); HiveOutputDescription outputDesc = new HiveOutputDescription(); outputDesc.getTableDesc().setTableName("foo"); OutputConf outputConf = new OutputConf(conf, PROFILE_ID); outputConf.writeOutputDescription(outputDesc); HiveApiOutputFormat outputFormat = new HiveApiOutputFormat(); outputFormat.setMyProfileId(PROFILE_ID); JobConf jobConf = new JobConf(conf); TaskAttemptContext taskContext = new HackTaskAttemptContext(jobConf, new TaskAttemptID()); JobContext jobContext = new HackJobContext(jobConf, taskContext.getJobID()); HiveMetastores.setTestClient(metastore); outputFormat.checkOutputSpecs(jobContext); assertEquals( BackoffRetryTask.NUM_TRIES.getDefaultValue(), metastore.getNumCalls()); } }