Refine search
int origFactor = factor; int passNo = 1; LocalDirAllocator lDirAlloc = new LocalDirAllocator("io.seqfile.local.dir"); do { new Path(tmpDir, "intermediate").suffix("." + passNo); Path outputFile = lDirAlloc.getLocalPathForWrite( tmpFilename.toString(), approxOutputSize, conf); if(LOG.isDebugEnabled()) { fs.makeQualified(segmentsToMerge.get(0).segmentPathName), fs.makeQualified(outputFile), null); fs.getFileStatus(outputFile).getLen(), outputFile);
/** * Get the current directory index for the given configuration item. * @return the current directory index for the given configuration item. */ int getCurrentDirectoryIndex() { AllocatorPerContext context = obtainContext(contextCfgItemName); return context.getCurrentDirectoryIndex(); }
System.exit(1); JobConf conf = new JobConf(new Path(jobFilename.toString())); TaskAttemptID taskId = TaskAttemptID.forName(conf.get("mapred.task.id")); boolean isMap = conf.getBoolean("mapred.task.is.map", true); int partition = conf.getInt("mapred.task.partition", 0); FileSystem local = FileSystem.getLocal(conf); LocalDirAllocator lDirAlloc = new LocalDirAllocator("mapred.local.dir"); File workDirName = new File(lDirAlloc.getLocalPathToRead( TaskTracker.getLocalTaskDir( taskId.getJobID().toString(), taskId.toString()) + Path.SEPARATOR + "work", conf). toString()); local.setWorkingDirectory(new Path(workDirName.toString())); FileSystem.get(conf).setWorkingDirectory(conf.getWorkingDirectory());
private void localizeTask(Task task) throws IOException{ lDirAlloc.getLocalPathForWrite( TaskTracker.getLocalTaskDir(task.getJobID().toString(), task.getTaskID().toString(), task.isTaskCleanupTask()), defaultJobConf ); String workDir = lDirAlloc.getLocalPathToRead( TaskTracker.getLocalJobDir(task.getJobID().toString()) + Path.SEPARATOR + "work", defaultJobConf).toString(); String link = localTaskDir.getParent().toString() + Path.SEPARATOR + "work"; File flink = new File(link); Path cwd = lDirAlloc.getLocalPathForWrite( getLocalTaskDir(task.getJobID().toString(), task.getTaskID().toString(), task.isTaskCleanupTask()) + Path.SEPARATOR + MRConstants.WORKDIR, defaultJobConf); if (!localFs.mkdirs(cwd)) { localJobConf.set("mapred.local.dir", fConf.get("mapred.local.dir")); if (fConf.get("slave.host.name") != null) { localJobConf.set("mapred.task.id", task.getTaskID().toString());
LocalDirAllocator lDirAlloc = new LocalDirAllocator("mapred.local.dir"); File jobCacheDir = null; if (conf.getJar() != null) { jobCacheDir = new File( new Path(conf.getJar()).getParent().toString()); File workDir = new File(lDirAlloc.getLocalPathToRead( TaskTracker.getLocalTaskDir( t.getJobID().toString(), t.getTaskID().toString(), t.isTaskCleanupTask()) + Path.SEPARATOR + MRConstants.WORKDIR, conf). toString()); String cachePath = TaskTracker.getCacheSubdir() + Path.SEPARATOR + cacheId; if (lDirAlloc.ifExists(cachePath, conf)) { localPath = lDirAlloc.getLocalPathToRead(cachePath, conf); localPath = lDirAlloc.getLocalPathForWrite(cachePath, fileStatus.getLen(), conf); String cachePath = TaskTracker.getCacheSubdir() + Path.SEPARATOR + cacheId; if (lDirAlloc.ifExists(cachePath,conf)) { localPath = lDirAlloc.getLocalPathToRead(cachePath, conf); } else { localPath = lDirAlloc.getLocalPathForWrite(cachePath, fileStatus.getLen(), conf);
/** * Return the path to a local map output index file created earlier * * @return path * @throws IOException */ public Path getOutputIndexFile() throws IOException { Path attemptIndexOutput = new Path(getAttemptOutputDir(), MAP_OUTPUT_FILENAME_STRING + MAP_OUTPUT_INDEX_SUFFIX_STRING); return lDirAlloc.getLocalPathToRead(attemptIndexOutput.toString(), conf); }
String[] localSysDirs = StringUtils.getTrimmedStrings( System.getenv(Environment.LOCAL_DIRS.name())); job.setStrings(MRConfig.LOCAL_DIR, localSysDirs); LOG.info(MRConfig.LOCAL_DIR + " for child: " + job.get(MRConfig.LOCAL_DIR)); LocalDirAllocator lDirAlloc = new LocalDirAllocator(MRConfig.LOCAL_DIR); Path workDir = null; workDir = lDirAlloc.getLocalPathToRead("work", job); } catch (DiskErrorException e) { workDir = lDirAlloc.getLocalPathForWrite("work", job); FileSystem lfs = FileSystem.getLocal(job).getRaw(); boolean madeDir = false; try { madeDir = lfs.mkdirs(workDir); } catch (FileAlreadyExistsException e) { workDir = lDirAlloc.getLocalPathToRead("work", job); + workDir.toString()); job.set(MRJobConfig.JOB_LOCAL_DIR,workDir.toString());
FileSystem localFs = FileSystem.getLocal(fConf); getLocalizer().initializeJobDirs(userName, jobId); Token<JobTokenIdentifier> jt = TokenCache.getJobToken(ts); if (jt != null) { //could be null in the case of some unit tests getJobTokenSecretManager().addTokenForJob(jobId.toString(), jt); localizeJobConfFile(new Path(t.getJobFile()), userName, jobId); JobConf localJobConf = new JobConf(localJobFile); localJobConf.setUser(userName); localJobConf.set(TokenCache.JOB_TOKENS_FILENAME, localJobTokenFile); lDirAlloc.getLocalPathForWrite(getJobWorkDir(userName, jobId .toString()), fConf); if (!localFs.mkdirs(workDir)) { throw new IOException("Mkdirs failed to create " + workDir.toString()); System.setProperty(JOB_LOCAL_DIR, workDir.toUri().getPath()); localJobConf.set(JOB_LOCAL_DIR, workDir.toUri().getPath());
/** * Test getLocalPathToRead() returns correct filename and "file" schema. * * @throws IOException */ @Test (timeout = 30000) public void testGetLocalPathToRead() throws IOException { assumeTrue(!isWindows); String dir = buildBufferDir(ROOT, 0); try { conf.set(CONTEXT, dir); assertTrue(localFs.mkdirs(new Path(dir))); File f1 = dirAllocator.createTmpFileForWrite(FILENAME, SMALL_FILE_SIZE, conf); Path p1 = dirAllocator.getLocalPathToRead(f1.getName(), conf); assertEquals(f1.getName(), p1.getName()); assertEquals("file", p1.getFileSystem(conf).getUri().getScheme()); } finally { Shell.execCommand(Shell.getSetPermissionCommand("u+w", false, BUFFER_DIR_ROOT)); rmBufferDirs(); } }
public void cleanupTemporalDirectories() { if (deletionService == null) return; LocalDirAllocator lDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname); try { Iterable<Path> iter = lDirAllocator.getAllLocalPathsToRead(".", systemConf); FileSystem localFS = FileSystem.getLocal(systemConf); for (Path path : iter) { PathData[] items = PathData.expandAsGlob(localFS.makeQualified(new Path(path, "*")).toString(), systemConf); ArrayList<Path> paths = new ArrayList<Path>(); for (PathData pd : items) { paths.add(pd.path); } if (paths.size() == 0) continue; deletionService.delete(null, paths.toArray(new Path[paths.size()])); } } catch (IOException e) { LOG.error(e.getMessage(), e); } }
throws IOException, InterruptedException { String jarFile = localJobConf.getJar(); FileStatus status = null; long jarFileSize = -1; if (jarFile != null) { Path jarFilePath = new Path(jarFile); FileSystem fs = getFS(jarFilePath, jobId, localJobConf); try { status = fs.getFileStatus(jarFilePath); jarFileSize = status.getLen(); } catch (FileNotFoundException fe) { lDirAlloc.getLocalPathForWrite( getJobJarFile(user, jobId.toString()), 5 * jarFileSize, fConf); fs.copyToLocalFile(jarFilePath, localJarFile); localJobConf.setJar(localJarFile.toString()); new File(localJarFile.toString()), new File(localJarFile.getParent().toString()), localJobConf.getJarUnpackPattern());
/** * Create a local map output file name. * * @param size the size of the file * @return path * @throws IOException */ public Path getOutputFileForWrite(long size) throws IOException { Path attemptOutput = new Path(getAttemptOutputDir(), MAP_OUTPUT_FILENAME_STRING); return lDirAlloc.getLocalPathForWrite(attemptOutput.toString(), size, conf); }
public HashShuffleAppenderManager(TajoConf systemConf) throws IOException { this.systemConf = systemConf; // initialize LocalDirAllocator lDirAllocator = new LocalDirAllocator(ConfVars.WORKER_TEMPORAL_DIR.varname); // initialize DFS and LocalFileSystems defaultFS = TajoConf.getTajoRootDir(systemConf).getFileSystem(systemConf); localFS = FileSystem.getLocal(systemConf); pageSize = systemConf.getIntVar(ConfVars.SHUFFLE_HASH_APPENDER_PAGE_VOLUME) * StorageUnit.MB; Iterable<Path> allLocalPath = lDirAllocator.getAllLocalPathsToRead(".", systemConf); //add async hash shuffle writer for (Path path : allLocalPath) { temporalPaths.add(localFS.makeQualified(path).toString()); executors.put(temporalPaths.size() - 1, Executors.newSingleThreadExecutor()); } }
MapOutput(TaskAttemptID mapId, MergeManager<K,V> merger, long size, JobConf conf, LocalDirAllocator localDirAllocator, int fetcher, boolean primaryMapOutput) throws IOException { this.id = ID.incrementAndGet(); this.mapId = mapId; this.merger = merger; type = Type.DISK; memory = null; byteStream = null; this.size = size; this.localFS = FileSystem.getLocal(conf); String filename = "map_" + mapId.getTaskID().getId() + ".out"; String tmpOutput = Path.SEPARATOR + TaskTracker.getJobCacheSubdir(conf.getUser()) + Path.SEPARATOR + mapId.getJobID() + Path.SEPARATOR + merger.getReduceId() + Path.SEPARATOR + "output" + Path.SEPARATOR + filename + "." + fetcher; tmpOutputPath = localDirAllocator.getLocalPathForWrite(tmpOutput, size, conf); outputPath = new Path(tmpOutputPath.getParent(), filename); disk = localFS.create(tmpOutputPath); this.primaryMapOutput = primaryMapOutput; }
/** * Download the job-token file from the FS and save on local fs. * @param user * @param jobId * @param jobConf * @return the local file system path of the downloaded file. * @throws IOException */ private String localizeJobTokenFile(String user, JobID jobId) throws IOException { // check if the tokenJob file is there.. Path skPath = new Path(systemDirectory, jobId.toString()+"/"+TokenCache.JOB_TOKEN_HDFS_FILE); FileStatus status = null; long jobTokenSize = -1; status = systemFS.getFileStatus(skPath); //throws FileNotFoundException jobTokenSize = status.getLen(); Path localJobTokenFile = lDirAlloc.getLocalPathForWrite(getLocalJobTokenFile(user, jobId.toString()), jobTokenSize, fConf); String localJobTokenFileStr = localJobTokenFile.toUri().getPath(); LOG.debug("localizingJobTokenFile from sd="+skPath.toUri().getPath() + " to " + localJobTokenFileStr); // Download job_token systemFS.copyToLocalFile(skPath, localJobTokenFile); return localJobTokenFileStr; }
void localizeTask(Task task) throws IOException{ FileSystem localFs = FileSystem.getLocal(fConf); task.getJobID().toString(), task.getTaskID().toString(), task.isTaskCleanupTask()); lDirAlloc.getLocalPathForWrite(getTaskWorkDir(task.getUser(), task .getJobID().toString(), task.getTaskID().toString(), task .isTaskCleanupTask()), defaultJobConf); if (!localFs.mkdirs(cwd)) { throw new IOException("Mkdirs failed to create " + cwd.toString()); localJobConf.set(LOCAL_DIR, fConf.get(LOCAL_DIR)); if (fConf.get(TT_HOST_NAME) != null) { localJobConf.set(TT_HOST_NAME, fConf.get(TT_HOST_NAME)); if (keepPattern != null) { alwaysKeepTaskFiles = Pattern.matches(keepPattern, task.getTaskID().toString()); } else { alwaysKeepTaskFiles = false;
for (String dir : trackerFConf.getStrings(MRConfig.LOCAL_DIR)) { File attemptDir = new File(dir, TaskTracker.getLocalTaskDir(task.getUser(), jobId lDirAlloc.getLocalPathToRead(TaskTracker.getTaskWorkDir( task.getUser(), task.getJobID().toString(), task.getTaskID() .toString(), task.isTaskCleanupTask()), trackerFConf); assertTrue("atttempt work dir for " + taskId.toString() TaskRunner.createChildTmpDir(new File(attemptWorkDir.toUri().getPath()), localizedJobConf); attemptLogFiles = runner.prepareLogFiles(task.getTaskID(), lDirAlloc.getLocalPathToRead(TaskTracker.getTaskConfFile(task .getUser(), task.getJobID().toString(), task.getTaskID() .toString(), task.isTaskCleanupTask()), trackerFConf); assertTrue("Task conf file " + localTaskFile.toString() + " is not created!!", new File(localTaskFile.toUri().getPath()) .exists()); localizedTaskConf = new JobConf(localTaskFile); TaskRunner.setupChildMapredLocalDirs(task, localizedTaskConf); taskContext.env = new JvmEnv(null, null, null, null, -1, new File(localizedJobConf .get(TaskTracker.JOB_LOCAL_DIR)), null, localizedJobConf); taskContext.task = task;
public Path createBaseDir() throws IOException { // the base dir for an output dir String baseDir = getBaseOutputDir(executionBlockId).toString(); Path baseDirPath = localFS.makeQualified(getLocalDirAllocator().getLocalPathForWrite(baseDir, systemConf)); return baseDirPath; }
/** * Download the job configuration file from the FS. * * @param t Task whose job file has to be downloaded * @param jobId jobid of the task * @return the local file system path of the downloaded file. * @throws IOException */ private Path localizeJobConfFile(Path jobFile, String user, JobID jobId) throws IOException, InterruptedException { final JobConf conf = new JobConf(getJobConf()); FileSystem userFs = getFS(jobFile, jobId, conf); // Get sizes of JobFile // sizes are -1 if they are not present. FileStatus status = null; long jobFileSize = -1; try { status = userFs.getFileStatus(jobFile); jobFileSize = status.getLen(); } catch(FileNotFoundException fe) { jobFileSize = -1; } Path localJobFile = lDirAlloc.getLocalPathForWrite(getLocalJobConfFile(user, jobId.toString()), jobFileSize, fConf); // Download job.xml userFs.copyToLocalFile(jobFile, localJobFile); return localJobFile; }
/** Creates the working directory pathname for a task attempt. */ static File formWorkDir(LocalDirAllocator lDirAlloc, TaskAttemptID task, boolean isCleanup, JobConf conf) throws IOException { Path workDir = lDirAlloc.getLocalPathToRead(TaskTracker.getTaskWorkDir( conf.getUser(), task.getJobID().toString(), task.toString(), isCleanup), conf); return new File(workDir.toString()); }