private Path createScratchDir() throws IOException { Path parent = new Path(SessionState.get().getHdfsScratchDirURIString(), SPARK_DIR); Path sparkDir = new Path(parent, sessionId); FileSystem fs = sparkDir.getFileSystem(conf); FsPermission fsPermission = new FsPermission(HiveConf.getVar( conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION)); fs.mkdirs(sparkDir, fsPermission); fs.deleteOnExit(sparkDir); return sparkDir; }
private Path createScratchDir() throws IOException { Path parent = new Path(SessionState.get().getHdfsScratchDirURIString(), SPARK_DIR); Path sparkDir = new Path(parent, sessionId); FileSystem fs = sparkDir.getFileSystem(conf); FsPermission fsPermission = new FsPermission(HiveConf.getVar( conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION)); fs.mkdirs(sparkDir, fsPermission); fs.deleteOnExit(sparkDir); return sparkDir; }
/** * createTezDir creates a temporary directory in the scratchDir folder to * be used with Tez. Assumes scratchDir exists. */ private Path createTezDir(String sessionId) throws IOException { // tez needs its own scratch dir (per session) Path tezDir = new Path(SessionState.get().getHdfsScratchDirURIString(), TEZ_DIR); tezDir = new Path(tezDir, sessionId); FileSystem fs = tezDir.getFileSystem(conf); FsPermission fsPermission = new FsPermission(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION)); fs.mkdirs(tezDir, fsPermission); // Make sure the path is normalized (we expect validation to pass since we just created it). tezDir = DagUtils.validateTargetDir(tezDir, conf).getPath(); // Directory removal will be handled by cleanup at the SessionState level. return tezDir; }
/** * createTezDir creates a temporary directory in the scratchDir folder to * be used with Tez. Assumes scratchDir exists. */ private Path createTezDir(String sessionId, String suffix) throws IOException { // tez needs its own scratch dir (per session) // TODO: De-link from SessionState. A TezSession can be linked to different Hive Sessions via the pool. SessionState sessionState = SessionState.get(); String hdfsScratchDir = sessionState == null ? HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIR) : sessionState .getHdfsScratchDirURIString(); Path tezDir = new Path(hdfsScratchDir, TEZ_DIR); tezDir = new Path(tezDir, sessionId + ((suffix == null) ? "" : ("-" + suffix))); FileSystem fs = tezDir.getFileSystem(conf); FsPermission fsPermission = new FsPermission(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION)); fs.mkdirs(tezDir, fsPermission); // Make sure the path is normalized (we expect validation to pass since we just created it). tezDir = DagUtils.validateTargetDir(tezDir, conf).getPath(); // Directory removal will be handled by cleanup at the SessionState level. return tezDir; }
/** * createTezDir creates a temporary directory in the scratchDir folder to * be used with Tez. Assumes scratchDir exists. */ private Path createTezDir(String sessionId) throws IOException { // tez needs its own scratch dir (per session) Path tezDir = new Path(SessionState.get().getHdfsScratchDirURIString(), TEZ_DIR); tezDir = new Path(tezDir, sessionId); FileSystem fs = tezDir.getFileSystem(conf); FsPermission fsPermission = new FsPermission(HiveConf.getVar(conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION)); fs.mkdirs(tezDir, fsPermission); // Make sure the path is normalized (we expect validation to pass since we just created it). tezDir = DagUtils.validateTargetDir(tezDir, conf).getPath(); // don't keep the directory around on non-clean exit fs.deleteOnExit(tezDir); return tezDir; }