Refine search
reader = new BufferedReader(new InputStreamReader(xlearningProcess.getInputStream())); List<OutputInfo> outputs = Arrays.asList(amClient.getOutputLocation()); JobConf jobConf = new JobConf(conf); jobConf.setOutputKeyClass(Text.class); jobConf.setOutputValueClass(Text.class); jobConf.setBoolean("mapred.output.compress", true); jobConf.set("mapred.output.compression.codec", "org.apache.hadoop.io.compress.GzipCodec"); jobConf.setOutputFormat(TextMultiOutputFormat.class); Path remotePath = new Path(outputs.get(0).getDfsLocation() + "/_temporary/" + containerId.toString()); FileSystem dfs = remotePath.getFileSystem(jobConf); jobConf.set(XLearningConstants.STREAM_OUTPUT_DIR, remotePath.makeQualified(dfs).toString()); OutputFormat outputFormat = ReflectionUtils.newInstance(conf.getClass(XLearningConfiguration.XLEARNING_OUTPUTFORMAT_CLASS, XLearningConfiguration.DEFAULT_XLEARNING_OUTPUTF0RMAT_CLASS, OutputFormat.class), jobConf); outputFormat.checkOutputSpecs(dfs, jobConf); JobID jobID = new JobID(new SimpleDateFormat("yyyyMMddHHmm").format(new Date()), 0); TaskAttemptID taId = new TaskAttemptID(new TaskID(jobID, true, 0), 0); jobConf.set("mapred.tip.id", taId.getTaskID().toString()); jobConf.set("mapred.task.id", taId.toString()); jobConf.set("mapred.job.id", jobID.toString()); amClient.reportMapedTaskID(containerId, taId.toString()); RecordWriter writer = outputFormat.getRecordWriter(dfs, jobConf, "part-r", Reporter.NULL); String xlearningStreamResultLine; dfs.close(); } catch (Exception e) { LOG.warn("Exception in thread stdoutRedirectThread");
if (job.getBoolean(TetherJob.TETHER_EXEC_CACHED,false)){ localFiles = new Path[] { new Path(files[0].toString()) }; executable=localFiles[0].toString(); FileUtil.chmod(executable.toString(), "a+x"); executable=job.get(TetherJob.TETHER_EXEC); String args=job.get(TetherJob.TETHER_EXEC_ARGS); TaskAttemptID taskid = TaskAttemptID.forName(job.get("mapred.task.id")); File stdout = TaskLog.getTaskLogFile(taskid, false, TaskLog.LogName.STDOUT); File stderr = TaskLog.getTaskLogFile(taskid, false, TaskLog.LogName.STDERR);
/** * create the temporary output file for hadoop RecordWriter. * @param taskNumber The number of the parallel instance. * @param numTasks The number of parallel tasks. * @throws java.io.IOException */ @Override public void open(int taskNumber, int numTasks) throws IOException { // enforce sequential open() calls synchronized (OPEN_MUTEX) { if (Integer.toString(taskNumber + 1).length() > 6) { throw new IOException("Task id too large."); } TaskAttemptID taskAttemptID = TaskAttemptID.forName("attempt__0000_r_" + String.format("%" + (6 - Integer.toString(taskNumber + 1).length()) + "s", " ").replace(" ", "0") + Integer.toString(taskNumber + 1) + "_0"); this.jobConf.set("mapred.task.id", taskAttemptID.toString()); this.jobConf.setInt("mapred.task.partition", taskNumber + 1); // for hadoop 2.2 this.jobConf.set("mapreduce.task.attempt.id", taskAttemptID.toString()); this.jobConf.setInt("mapreduce.task.partition", taskNumber + 1); this.context = new TaskAttemptContextImpl(this.jobConf, taskAttemptID); this.outputCommitter = this.jobConf.getOutputCommitter(); JobContext jobContext = new JobContextImpl(this.jobConf, new JobID()); this.outputCommitter.setupJob(jobContext); this.recordWriter = this.mapredOutputFormat.getRecordWriter(null, this.jobConf, Integer.toString(taskNumber + 1), new HadoopDummyProgressable()); } }
public static TaskID getTaskID(Configuration cfg) { // first try with the attempt since some Hadoop versions mix the two String taskAttemptId = HadoopCfgUtils.getTaskAttemptId(cfg); if (StringUtils.hasText(taskAttemptId)) { try { return TaskAttemptID.forName(taskAttemptId).getTaskID(); } catch (IllegalArgumentException ex) { // the task attempt is invalid (Tez in particular uses the wrong string - see #346) // try to fallback to task id return parseTaskIdFromTaskAttemptId(taskAttemptId); } } String taskIdProp = HadoopCfgUtils.getTaskId(cfg); // double-check task id bug in Hadoop 2.5.x if (StringUtils.hasText(taskIdProp) && !taskIdProp.contains("attempt")) { return TaskID.forName(taskIdProp); } return null; }
Path getWorkPath(TaskAttemptContext taskContext, Path basePath) throws IOException { // ${mapred.out.dir}/_temporary Path jobTmpDir = new Path(basePath, FileOutputCommitter.TEMP_DIR_NAME); FileSystem fs = jobTmpDir.getFileSystem(taskContext.getJobConf()); if (!fs.exists(jobTmpDir)) { throw new IOException("The temporary job-output directory " + jobTmpDir.toString() + " doesn't exist!"); } // ${mapred.out.dir}/_temporary/_${taskid} String taskid = taskContext.getTaskAttemptID().toString(); Path taskTmpDir = new Path(jobTmpDir, "_" + taskid); if (!fs.mkdirs(taskTmpDir)) { throw new IOException("Mkdirs failed to create " + taskTmpDir.toString()); } return taskTmpDir; } }
DOMConfigurator.configure("log4j.xml"); FileSystem fs = FileSystem.get(job.getConfiguration()); Path execBasePath = new Path(props.getProperty(ETL_EXECUTION_BASE_PATH)); Path execHistory = new Path(props.getProperty(ETL_EXECUTION_HISTORY_PATH)); if (!fs.exists(execBasePath)) { log.info("The execution base path does not exist. Creating the directory"); fs.mkdirs(execBasePath); if (!fs.exists(execHistory)) { log.info("removing old execution: " + stat.getPath().getName()); ContentSummary execContent = fs.getContentSummary(stat.getPath()); currentCount -= execContent.getFileCount() + execContent.getDirectoryCount(); JobClient client = new JobClient(new JobConf(job.getConfiguration())); for (TaskReport task : client.getMapTaskReports(tasks[0].getTaskAttemptId().getJobID())) { if (task.getCurrentStatus().equals(TIPStatus.FAILED)) { for (String s : task.getDiagnostics()) {
System.exit(1); JobConf conf = new JobConf(new Path(jobFilename.toString())); TaskAttemptID taskId = TaskAttemptID.forName(conf.get("mapred.task.id")); boolean isMap = conf.getBoolean("mapred.task.is.map", true); int partition = conf.getInt("mapred.task.partition", 0); FileSystem local = FileSystem.getLocal(conf); LocalDirAllocator lDirAlloc = new LocalDirAllocator("mapred.local.dir"); File workDirName = new File(lDirAlloc.getLocalPathToRead( TaskTracker.getLocalTaskDir( taskId.getJobID().toString(), taskId.toString()) + Path.SEPARATOR + "work", conf). toString()); local.setWorkingDirectory(new Path(workDirName.toString())); FileSystem.get(conf).setWorkingDirectory(conf.getWorkingDirectory());
private void localizeTask(Task task) throws IOException{ lDirAlloc.getLocalPathForWrite( TaskTracker.getLocalTaskDir(task.getJobID().toString(), task.getTaskID().toString(), task.isTaskCleanupTask()), defaultJobConf ); Path cwd = lDirAlloc.getLocalPathForWrite( getLocalTaskDir(task.getJobID().toString(), task.getTaskID().toString(), task.isTaskCleanupTask()) + Path.SEPARATOR + MRConstants.WORKDIR, defaultJobConf); if (!localFs.mkdirs(cwd)) { localJobConf.set("mapred.local.dir", fConf.get("mapred.local.dir")); if (fConf.get("slave.host.name") != null) { localJobConf.set("mapred.task.id", task.getTaskID().toString()); if (keepPattern != null) { alwaysKeepTaskFiles = Pattern.matches(keepPattern, task.getTaskID().toString()); } else { alwaysKeepTaskFiles = false;
private void testMapOnlyNoOutputInternal(int version) throws Exception { JobConf conf = new JobConf(); //This is not set on purpose. FileOutputFormat.setOutputPath(conf, outDir); conf.set(JobContext.TASK_ATTEMPT_ID, attempt); conf.setInt(org.apache.hadoop.mapreduce.lib.output. FileOutputCommitter.FILEOUTPUTCOMMITTER_ALGORITHM_VERSION, version); JobContext jContext = new JobContextImpl(conf, taskID.getJobID()); TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID); FileOutputCommitter committer = new FileOutputCommitter(); // setup committer.setupJob(jContext); committer.setupTask(tContext); if(committer.needsTaskCommit(tContext)) { // do commit committer.commitTask(tContext); } committer.commitJob(jContext); // validate output FileUtil.fullyDelete(new File(outDir.toString())); }
/** * Obtain the owner of the log dir. This is * determined by checking the job's log directory. */ static String obtainLogDirOwner(TaskAttemptID taskid) throws IOException { Configuration conf = new Configuration(); FileSystem raw = FileSystem.getLocal(conf).getRaw(); Path jobLogDir = new Path(getJobDir(taskid.getJobID()).getAbsolutePath()); FileStatus jobStat = raw.getFileStatus(jobLogDir); return jobStat.getOwner(); }
@SuppressWarnings("unchecked") public void testCommitter() throws Exception { JobConf job = new JobConf(); setConfForFileOutputCommitter(job); JobContext jContext = new JobContextImpl(job, taskID.getJobID()); TaskAttemptContext tContext = new TaskAttemptContextImpl(job, taskID); FileOutputCommitter committer = new FileOutputCommitter(); FileSystem localFs = FileSystem.getLocal(job); TextOutputFormat theOutputFormat = new TextOutputFormat(); RecordWriter theRecordWriter = File expectedFile = new File(new Path(outDir, file).toString()); StringBuffer expectedOutput = new StringBuffer(); expectedOutput.append(key1).append('\t').append(val1).append("\n");
private void testFailAbortInternal(int version) throws IOException, InterruptedException { JobConf conf = new JobConf(); conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "faildel:///"); conf.setClass("fs.faildel.impl", FakeFileSystem.class, FileSystem.class); conf.set(JobContext.TASK_ATTEMPT_ID, attempt); conf.setInt(org.apache.hadoop.mapreduce.lib.output. conf.setInt(MRConstants.APPLICATION_ATTEMPT_ID, 1); FileOutputFormat.setOutputPath(conf, outDir); JobContext jContext = new JobContextImpl(conf, taskID.getJobID()); TaskAttemptContext tContext = new TaskAttemptContextImpl(conf, taskID); FileOutputCommitter committer = new FileOutputCommitter(); File jobTmpDir = new File(new Path(outDir, FileOutputCommitter.TEMP_DIR_NAME + Path.SEPARATOR + conf.getInt(MRConstants.APPLICATION_ATTEMPT_ID, 0) + Path.SEPARATOR + FileOutputCommitter.TEMP_DIR_NAME).toString()); File taskTmpDir = new File(jobTmpDir, "_" + taskID); File expectedFile = new File(taskTmpDir, partFile);
TaskAttemptID.getTaskAttemptIDsPattern(null, null, TaskType.MAP, 1, null); jobConf.setKeepTaskFilesPattern(pattern); TestResult result; final Path inDir = new Path("./wc/input"); final Path outDir = new Path("./wc/output"); String input = "The quick brown fox\nhas many silly\nred fox sox\n"; result = launchWordCount(jobConf, inDir, outDir, input, 3, 1); "quick\t1\nred\t1\nsilly\t1\nsox\t1\n", result.output); JobID jobid = result.job.getID(); TaskAttemptID taskid = new TaskAttemptID( new TaskID(jobid, TaskType.MAP, 1),0); String userName = UserGroupInformation.getLoginUser().getUserName(); checkTaskDirectories(mr, userName, new String[] { jobid.toString() }, new String[] { taskid.toString() }); FileSystem localfs = FileSystem.getLocal(jobConf); String TEST_ROOT_DIR = new File(System.getProperty("test.build.data","/tmp")) .toString().replace(' ', '+'); Path localIn = localfs.makeQualified (new Path(TEST_ROOT_DIR + "/local/in")); Path localOut = localfs.makeQualified (new Path(TEST_ROOT_DIR + "/local/out")); result = launchWordCount(jobConf, localIn, localOut,
@Override public void configure(JobConf conf) { try { signalFileDir = new Path(conf.get("signal.dir.path")); numReducers = conf.getNumReduceTasks(); fs = FileSystem.get(conf); String taskAttemptId = conf.get(JobContext.TASK_ATTEMPT_ID); if (taskAttemptId != null) { TaskAttemptID taskAttemptID = TaskAttemptID.forName(taskAttemptId); taskNumber = taskAttemptID.getTaskID().getId(); } } catch (IOException ioe) { LOG.warn("Caught exception " + ioe); } }
void localizeTask(Task task) throws IOException{ FileSystem localFs = FileSystem.getLocal(fConf); task.getJobID().toString(), task.getTaskID().toString(), task.isTaskCleanupTask()); lDirAlloc.getLocalPathForWrite(getTaskWorkDir(task.getUser(), task .getJobID().toString(), task.getTaskID().toString(), task .isTaskCleanupTask()), defaultJobConf); if (!localFs.mkdirs(cwd)) { throw new IOException("Mkdirs failed to create " + cwd.toString()); localJobConf.set(LOCAL_DIR, fConf.get(LOCAL_DIR)); if (fConf.get(TT_HOST_NAME) != null) { localJobConf.set(TT_HOST_NAME, fConf.get(TT_HOST_NAME)); if (keepPattern != null) { alwaysKeepTaskFiles = Pattern.matches(keepPattern, task.getTaskID().toString()); } else { alwaysKeepTaskFiles = false;
@SuppressWarnings("rawtypes") public MockMapredOutputFormat(JobConf outputFormatConf, Class<? extends OutputFormat> outputFormatClass, Class<? extends InputFormat> inputFormatClass, JobConf inputFormatConf) throws IOException { this.outputFormatConf = outputFormatConf; this.inputFormatConf = inputFormatConf; outputFormat = ReflectionUtils.newInstance(outputFormatClass, outputFormatConf); inputFormat = ReflectionUtils .newInstance(inputFormatClass, inputFormatConf); if (outputPath.exists()) { throw new IllegalStateException( "Generated the same random dir name twice: " + outputPath); } if (!outputPath.mkdir()) { throw new IOException("Failed to create output dir " + outputPath); } FileOutputFormat.setOutputPath(outputFormatConf, new Path(outputPath.toString())); outputFormatConf.set("mapred.task.id", TASK_ID.toString()); FileSystem.getLocal(outputFormatConf).mkdirs( new Path(outputPath.toString(), FileOutputCommitter.TEMP_DIR_NAME)); }
/** * Localize the given JobConf to be specific for this task. */ public void localizeConfiguration(JobConf conf) throws IOException { conf.set(JobContext.TASK_ID, taskId.getTaskID().toString()); conf.set(JobContext.TASK_ATTEMPT_ID, taskId.toString()); conf.setBoolean(JobContext.TASK_ISMAP, isMapTask()); conf.setInt(JobContext.TASK_PARTITION, partition); conf.set(JobContext.ID, taskId.getJobID().toString()); }
@Test public void testShufflePermissions() throws Exception { JobConf conf = new JobConf(); conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077"); conf.set(MRConfig.LOCAL_DIR, TEST_ROOT_DIR.getAbsolutePath()); MapOutputFile mof = new MROutputFiles(); mof.setConf(conf); TaskAttemptID attemptId = new TaskAttemptID("12345", 1, TaskType.MAP, 1, 1); MapTask mockTask = mock(MapTask.class); doReturn(mof).when(mockTask).getMapOutputFile(); mob.close(); Path outputFile = mof.getOutputFile(); FileSystem lfs = FileSystem.getLocal(conf); FsPermission perms = lfs.getFileStatus(outputFile).getPermission(); Assert.assertEquals("Incorrect output file perms", (short)0640, perms.toShort()); Path indexFile = mof.getOutputIndexFile(); perms = lfs.getFileStatus(indexFile).getPermission(); Assert.assertEquals("Incorrect index file perms", (short)0640, perms.toShort());
/** * Create empty sequence files for any of the map outputs that we don't have. * @param fs the filesystem to create the files in * @param dir the directory name to create the files in * @param conf the jobconf * @throws IOException if something goes wrong writing */ private static void fillInMissingMapOutputs(FileSystem fs, TaskAttemptID taskId, int numMaps, JobConf conf) throws IOException { Class<? extends WritableComparable> keyClass = conf.getMapOutputKeyClass().asSubclass(WritableComparable.class); Class<? extends Writable> valueClass = conf.getMapOutputValueClass().asSubclass(Writable.class); MapOutputFile namer = new MapOutputFile(taskId.getJobID()); namer.setConf(conf); for(int i=0; i<numMaps; i++) { Path f = namer.getInputFile(i, taskId); if (!fs.exists(f)) { LOG.info("Create missing input: " + f); SequenceFile.Writer out = SequenceFile.createWriter(fs, conf, f, keyClass, valueClass); out.close(); } } }