/** * On Tez we're not creating dummy files when getting/setting input paths. * We let Tez handle the situation. We're also setting the paths in the AM * so we don't want to depend on scratch dir and context. */ public static List<Path> getInputPathsTez(JobConf job, MapWork work) throws Exception { String scratchDir = job.get(DagUtils.TEZ_TMP_DIR_KEY); List<Path> paths = getInputPaths(job, work, new Path(scratchDir), null, true); return paths; }
/** * On Tez we're not creating dummy files when getting/setting input paths. * We let Tez handle the situation. We're also setting the paths in the AM * so we don't want to depend on scratch dir and context. */ public static List<Path> getInputPathsTez(JobConf job, MapWork work) throws Exception { String scratchDir = job.get(DagUtils.TEZ_TMP_DIR_KEY); List<Path> paths = getInputPaths(job, work, new Path(scratchDir), null, true); return paths; }
List<Path> inputPaths = Utilities.getInputPaths(cloned, (MapWork) work, scratchDir, context, false); Utilities.setInputPaths(cloned, inputPaths);
List<Path> inputPaths = Utilities.getInputPaths(jobConf, mapWork, scratchDir, mock(Context.class), false); assertEquals(inputPaths.size(), numOfPartitions); for (int i=0; i<numOfPartitions; i++) {
MapWork mapWork = (MapWork) work; cloned.setBoolean("mapred.task.is.map", true); List<Path> inputPaths = Utilities.getInputPaths(cloned, mapWork, scratchDir, context, false); Utilities.setInputPaths(cloned, inputPaths);
Path scratchDir = new Path(HiveConf.getVar(jobConf, HiveConf.ConfVars.LOCALSCRATCHDIR)); List<Path> inputPaths1 = Utilities.getInputPaths(jobConf, mapWork1, scratchDir, mock(Context.class), false); inputPaths.addAll(inputPaths1); assertFalse(nonExistentPath1.getFileSystem(conf).exists(nonExistentPath1)); List<Path> inputPaths2 = Utilities.getInputPaths(jobConf, mapWork2, scratchDir, mock(Context.class), false); inputPaths.addAll(inputPaths2);
List<Path> inputPaths = Utilities.getInputPaths(jobConf, mapWork, new Path(HiveConf.getVar(jobConf, HiveConf.ConfVars.LOCALSCRATCHDIR)), mock(Context.class), false); assertEquals(inputPaths.size(), numPartitions);
MapRedTask selectTask = (MapRedTask)plan.getRootTasks().get(0); List<Path> inputPaths = Utilities.getInputPaths(newJob, selectTask.getWork().getMapWork(), emptyScratchDir, ctx, false); Utilities.setInputPaths(newJob, inputPaths);
List<Path> inputPaths = Utilities.getInputPaths(job, mWork, emptyScratchDir, ctx, false); Utilities.setInputPaths(job, inputPaths);
List<Path> inputPaths = Utilities.getInputPaths(job, mWork, emptyScratchDir, ctx, false); Utilities.setInputPaths(job, inputPaths);
/** * On Tez we're not creating dummy files when getting/setting input paths. * We let Tez handle the situation. We're also setting the paths in the AM * so we don't want to depend on scratch dir and context. */ public static List<Path> getInputPathsTez(JobConf job, MapWork work) throws Exception { String scratchDir = job.get(DagUtils.TEZ_TMP_DIR_KEY); // we usually don't want to create dummy files for tez, however the metadata only // optimization relies on it. List<Path> paths = getInputPaths(job, work, new Path(scratchDir), null, !work.isUseOneNullRowInputFormat()); return paths; }
List<Path> inputPaths = Utilities.getInputPaths(cloned, (MapWork) work, scratchDir, context, false); Utilities.setInputPaths(cloned, inputPaths);
List<Path> inputPaths = Utilities.getInputPaths(job, mWork, emptyScratchDir, ctx, false); Utilities.setInputPaths(job, inputPaths);