@Test /** * A testing method verifying availability and accessibility of API needed for * AuxiliaryService(s) which are "Shuffle-Providers" (ShuffleHandler and 3rd party plugins) */ public void testProviderApi() { LocalDirAllocator mockLocalDirAllocator = mock(LocalDirAllocator.class); JobConf mockJobConf = mock(JobConf.class); try { mockLocalDirAllocator.getLocalPathToRead("", mockJobConf); } catch (Exception e) { assertTrue("Threw exception:" + e, false); } } }
/** * Return the path to a local map output index file created earlier * * @return path * @throws IOException */ public Path getOutputIndexFile() throws IOException { return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + Path.SEPARATOR + "file.out.index", conf); }
/** * Return a local map spill file created earlier. * * @param spillNumber the number * @return path * @throws IOException */ public Path getSpillFile(int spillNumber) throws IOException { return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + "/spill" + spillNumber + ".out", conf); }
/** * Return a local map spill index file created earlier * * @param spillNumber the number * @return path * @throws IOException */ public Path getSpillIndexFile(int spillNumber) throws IOException { return lDirAlloc.getLocalPathToRead(TaskTracker.OUTPUT + "/spill" + spillNumber + ".out.index", conf); }
/** * Return a local map spill index file created earlier * * @param spillNumber the number * @return path * @throws IOException */ public Path getSpillIndexFile(int spillNumber) throws IOException { return lDirAlloc.getLocalPathToRead( String.format(SPILL_INDEX_FILE_PATTERN, conf.get(JobContext.TASK_ATTEMPT_ID), spillNumber), conf); }
/** * Return a local map spill file created earlier. * * @param spillNumber the number * @return path * @throws IOException */ public Path getSpillFile(int spillNumber) throws IOException { return lDirAlloc.getLocalPathToRead( String.format(SPILL_FILE_PATTERN, conf.get(JobContext.TASK_ATTEMPT_ID), spillNumber), conf); }
/** * Return a local map spill index file created earlier * * @param spillNumber the number * @return path * @throws IOException */ public Path getSpillIndexFile(int spillNumber) throws IOException { return lDirAlloc.getLocalPathToRead( String.format(SPILL_INDEX_FILE_PATTERN, conf.get(JobContext.TASK_ATTEMPT_ID), spillNumber), conf); }
/** * Return a local map spill file created earlier. * * @param spillNumber the number * @return path * @throws IOException */ public Path getSpillFile(int spillNumber) throws IOException { return lDirAlloc.getLocalPathToRead( String.format(SPILL_FILE_PATTERN, conf.get(JobContext.TASK_ATTEMPT_ID), spillNumber), conf); }
/** * Return the path to a local map output index file created earlier * * @return path * @throws IOException */ public Path getOutputIndexFile() throws IOException { Path attemptIndexOutput = new Path(getAttemptOutputDir(), MAP_OUTPUT_FILENAME_STRING + MAP_OUTPUT_INDEX_SUFFIX_STRING); return lDirAlloc.getLocalPathToRead(attemptIndexOutput.toString(), conf); }
/** * Return the path to local map output file created earlier * * @return path * @throws IOException */ @Override public Path getOutputFile() throws IOException { return lDirAlloc.getLocalPathToRead(MRJobConfig.OUTPUT + Path.SEPARATOR + MAP_OUTPUT_FILENAME_STRING, getConf()); }
/** * Return the path to a local map output index file created earlier * * @return path * @throws IOException */ public Path getOutputIndexFile() throws IOException { Path attemptIndexOutput = new Path(getAttemptOutputDir(), MAP_OUTPUT_FILENAME_STRING + MAP_OUTPUT_INDEX_SUFFIX_STRING); return lDirAlloc.getLocalPathToRead(attemptIndexOutput.toString(), conf); }
/** * Return the path to local map output file created earlier * * @return path * @throws IOException */ public Path getOutputFile() throws IOException { Path attemptOutput = new Path(getAttemptOutputDir(), MAP_OUTPUT_FILENAME_STRING); return lDirAlloc.getLocalPathToRead(attemptOutput.toString(), conf); }
@VisibleForTesting //TODO: Refactor following to make use of methods from TezTaskOutputFiles to be consistent. protected Path getShuffleInputFileName(String pathComponent, String suffix) throws IOException { LocalDirAllocator localDirAllocator = new LocalDirAllocator(TezRuntimeFrameworkConfigs.LOCAL_DIRS); suffix = suffix != null ? suffix : ""; String outputPath = Constants.TEZ_RUNTIME_TASK_OUTPUT_DIR + Path.SEPARATOR + pathComponent + Path.SEPARATOR + Constants.TEZ_RUNTIME_TASK_OUTPUT_FILENAME_STRING + suffix; String pathFromLocalDir = getPathForLocalDir(outputPath); return localDirAllocator.getLocalPathToRead(pathFromLocalDir.toString(), conf); }
/** Return the path to a local map output index file created earlier * @param mapTaskId a map task id */ public Path getOutputIndexFile(TaskAttemptID mapTaskId) throws IOException { return lDirAlloc.getLocalPathToRead(TaskTracker.getIntermediateOutputDir( jobId.toString(), mapTaskId.toString()) + "/file.out.index", conf); }
/** Return a local map spill index file created earlier * @param mapTaskId a map task id * @param spillNumber the number */ public Path getSpillIndexFile(TaskAttemptID mapTaskId, int spillNumber) throws IOException { return lDirAlloc.getLocalPathToRead(TaskTracker.getIntermediateOutputDir( jobId.toString(), mapTaskId.toString()) + "/spill" + spillNumber + ".out.index", conf); }
/** Return the path to local map output file created earlier * @param mapTaskId a map task id */ public Path getOutputFile(TaskAttemptID mapTaskId) throws IOException { return lDirAlloc.getLocalPathToRead(TaskTracker.getIntermediateOutputDir( jobId.toString(), mapTaskId.toString()) + "/file.out", conf); }
/** Creates the working directory pathname for a task attempt. */ static File formWorkDir(LocalDirAllocator lDirAlloc, TaskAttemptID task, boolean isCleanup, JobConf conf) throws IOException { Path workDir = lDirAlloc.getLocalPathToRead(TaskTracker.getTaskWorkDir( conf.getUser(), task.getJobID().toString(), task.toString(), isCleanup), conf); return new File(workDir.toString()); }
/** Return a local map spill index file created earlier * @param mapTaskId a map task id * @param spillNumber the number */ public Path getSpillIndexFile(TaskAttemptID mapTaskId, int spillNumber) throws IOException { return lDirAlloc.getLocalPathToRead(TaskTracker.getIntermediateOutputDir( jobId.toString(), mapTaskId.toString()) + "/spill" + spillNumber + ".out.index", conf); }
/** Return a local reduce input file created earlier * @param mapTaskId a map task id * @param reduceTaskId a reduce task id */ public Path getInputFile(int mapId, TaskAttemptID reduceTaskId) throws IOException { // TODO *oom* should use a format here return lDirAlloc.getLocalPathToRead(TaskTracker.getIntermediateOutputDir( jobId.toString(), reduceTaskId.toString()) + "/map_" + mapId + ".out", conf); }
private Path getAttemptJobXml(JobConf conf, JobID jobId, TaskType taskType) throws IOException, LoginException { String taskid = new TaskAttemptID(new TaskID(jobId, taskType, 0), 0).toString(); return new LocalDirAllocator(MRConfig.LOCAL_DIR).getLocalPathToRead( TaskTracker.getTaskConfFile(UserGroupInformation.getCurrentUser() .getUserName(), jobId.toString(), taskid, false), conf); }