Path tmpPath = ctx.getMRTmpPath(); if (comparePathKeyStrength(tablePath, tmpPath) < 0) { throw new HiveException("Read-only encrypted tables cannot be read " + stagingPath = ctx.getMRTmpPath(tablePath.toUri()); stagingPath = ctx.getMRTmpPath();
Path tmpPath = ctx.getMRTmpPath(); if (comparePathKeyStrength(tablePath, tmpPath) < 0) { throw new HiveException("Read-only encrypted tables cannot be read " + stagingPath = ctx.getMRTmpPath(tablePath.toUri()); stagingPath = ctx.getMRTmpPath();
conf.set(TEZ_TMP_DIR_KEY, context.getMRTmpPath().toUri().toString()); conf.set("mapred.mapper.class", ExecMapper.class.getName()); conf.set("mapred.input.format.class", inpFormat);
String tmpFile = pctx.getContext().getMRTmpPath().toUri().toString(); qlCommand.append( "\"" + tmpFile + "\" "); // QL includes " around file name qlCommand.append("SELECT bucketname AS `_bucketname` , COLLECT_SET(offset) AS `_offsets` FROM ");
conf.set(TEZ_TMP_DIR_KEY, context.getMRTmpPath().toUri().toString()); conf.set("mapred.mapper.class", ExecMapper.class.getName()); conf.set("mapred.input.format.class", inpFormat);
private SparkJobRef submit(final DriverContext driverContext, final SparkWork sparkWork) throws Exception { final Context ctx = driverContext.getCtx(); final HiveConf hiveConf = (HiveConf) ctx.getConf(); refreshLocalResources(sparkWork, hiveConf); final JobConf jobConf = new JobConf(hiveConf); //update the credential provider location in the jobConf HiveConfUtil.updateJobCredentialProviders(jobConf); // Create temporary scratch dir final Path emptyScratchDir = ctx.getMRTmpPath(); FileSystem fs = emptyScratchDir.getFileSystem(jobConf); fs.mkdirs(emptyScratchDir); // make sure NullScanFileSystem can be loaded - HIVE-18442 jobConf.set("fs." + NullScanFileSystem.getBaseScheme() + ".impl", NullScanFileSystem.class.getCanonicalName()); byte[] jobConfBytes = KryoSerializer.serializeJobConf(jobConf); byte[] scratchDirBytes = KryoSerializer.serialize(emptyScratchDir); byte[] sparkWorkBytes = KryoSerializer.serialize(sparkWork); JobStatusJob job = new JobStatusJob(jobConfBytes, scratchDirBytes, sparkWorkBytes); if (driverContext.isShutdown()) { throw new HiveException("Operation is cancelled."); } JobHandle<Serializable> jobHandle = remoteClient.submit(job); RemoteSparkJobStatus sparkJobStatus = new RemoteSparkJobStatus(remoteClient, jobHandle, sparkClientTimtout); return new RemoteSparkJobRef(hiveConf, jobHandle, sparkJobStatus); }
/** * Create a temporary directory depending of the path specified. * - If path is an Object store filesystem, then use the default MR scratch directory (HDFS), unless isFinalJob and * {@link BlobStorageUtils#areOptimizationsEnabled(Configuration)} are both true, then return a path on * the blobstore. * - If path is on HDFS, then create a staging directory inside the path * * @param path Path used to verify the Filesystem to use for temporary directory * * @return A path to the new temporary directory */ public Path getTempDirForInterimJobPath(Path path) { // For better write performance, we use HDFS for temporary data when object store is used. // Note that the scratch directory configuration variable must use HDFS or any other // non-blobstorage system to take advantage of this performance. boolean isBlobstorageOptimized = BlobStorageUtils.isBlobStoragePath(conf, path) && !BlobStorageUtils.isBlobStorageAsScratchDir(conf) && BlobStorageUtils.areOptimizationsEnabled(conf); if (isPathLocal(path) || isBlobstorageOptimized) { return getMRTmpPath(); } return getExtTmpPathRelTo(path); }
Path baseTmpPath = context.parseContext.getContext().getMRTmpPath(); tmpPath = SparkUtilities.generateTmpPathForPartitionPruning(baseTmpPath, targetId); targetWork.setTmpPathForPartitionPruning(tmpPath);
/** * Create a temporary directory depending of the path specified. * - If path is an Object store filesystem, then use the default MR scratch directory (HDFS), unless isFinalJob and * {@link BlobStorageUtils#areOptimizationsEnabled(Configuration)} are both true, then return a path on * the blobstore. * - If path is on HDFS, then create a staging directory inside the path * * @param path Path used to verify the Filesystem to use for temporary directory * @param isFinalJob true if the required {@link Path} will be used for the final job (e.g. the final FSOP) * * @return A path to the new temporary directory */ public Path getTempDirForPath(Path path, boolean isFinalJob) { if (((BlobStorageUtils.isBlobStoragePath(conf, path) && !BlobStorageUtils.isBlobStorageAsScratchDir( conf)) || isPathLocal(path))) { if (!(isFinalJob && BlobStorageUtils.areOptimizationsEnabled(conf))) { // For better write performance, we use HDFS for temporary data when object store is used. // Note that the scratch directory configuration variable must use HDFS or any other non-blobstorage system // to take advantage of this performance. return getMRTmpPath(); } } return getExtTmpPathRelTo(path); }
Path taskTmpDir = baseCtx.getMRTmpPath();
private SparkJobRef submit(final DriverContext driverContext, final SparkWork sparkWork) throws Exception { final Context ctx = driverContext.getCtx(); final HiveConf hiveConf = (HiveConf) ctx.getConf(); refreshLocalResources(sparkWork, hiveConf); final JobConf jobConf = new JobConf(hiveConf); //update the credential provider location in the jobConf HiveConfUtil.updateJobCredentialProviders(jobConf); // Create temporary scratch dir final Path emptyScratchDir = ctx.getMRTmpPath(); FileSystem fs = emptyScratchDir.getFileSystem(jobConf); fs.mkdirs(emptyScratchDir); byte[] jobConfBytes = KryoSerializer.serializeJobConf(jobConf); byte[] scratchDirBytes = KryoSerializer.serialize(emptyScratchDir); byte[] sparkWorkBytes = KryoSerializer.serialize(sparkWork); JobStatusJob job = new JobStatusJob(jobConfBytes, scratchDirBytes, sparkWorkBytes); if (driverContext.isShutdown()) { throw new HiveException("Operation is cancelled."); } JobHandle<Serializable> jobHandle = remoteClient.submit(job); RemoteSparkJobStatus sparkJobStatus = new RemoteSparkJobStatus(remoteClient, jobHandle, sparkClientTimtout); return new RemoteSparkJobRef(hiveConf, jobHandle, sparkJobStatus); }
Path taskTmpDir = baseCtx.getMRTmpPath();
@Test public void testGetScratchDirectoriesForPaths() throws IOException { Context spyContext = spy(context); // When Object store paths are used, then getMRTmpPatch() is called to get a temporary // directory on the default scratch diretory location (usually /temp) Path mrTmpPath = new Path("hdfs://hostname/tmp/scratch"); doReturn(mrTmpPath).when(spyContext).getMRTmpPath(); assertEquals(mrTmpPath, spyContext.getTempDirForInterimJobPath(new Path("s3a://bucket/dir"))); // When local filesystem paths are used, then getMRTmpPatch() should be called to // get a temporary directory assertEquals(mrTmpPath, spyContext.getTempDirForInterimJobPath(new Path("file:/user"))); assertEquals(mrTmpPath, spyContext.getTempDirForInterimJobPath(new Path("file:///user"))); // When Non-Object store paths are used, then getExtTmpPathRelTo is called to get a temporary // directory on the same path passed as a parameter Path tmpPathRelTo = new Path("hdfs://hostname/user"); doReturn(tmpPathRelTo).when(spyContext).getExtTmpPathRelTo(any(Path.class)); assertEquals(tmpPathRelTo, spyContext.getTempDirForInterimJobPath(new Path("/user"))); conf.setBoolean(HiveConf.ConfVars.HIVE_BLOBSTORE_OPTIMIZATIONS_ENABLED.varname, false); assertEquals(tmpPathRelTo, spyContext.getTempDirForInterimJobPath(new Path("s3a://bucket/dir"))); assertEquals(mrTmpPath, spyContext.getTempDirForInterimJobPath(new Path("file:///user"))); conf.setBoolean(HiveConf.ConfVars.HIVE_BLOBSTORE_OPTIMIZATIONS_ENABLED.varname, true); } }
emptyScratchDir = ctx.getMRTmpPath(); FileSystem fs = emptyScratchDir.getFileSystem(jobConf); fs.mkdirs(emptyScratchDir);
emptyScratchDir = ctx.getMRTmpPath(); FileSystem fs = emptyScratchDir.getFileSystem(jobConf); fs.mkdirs(emptyScratchDir);
Path taskTmpDir = baseCtx.getMRTmpPath();
Path taskTmpDir = baseCtx.getMRTmpPath(); Operator<? extends OperatorDesc> rsParent = reduceSink.getParentOperators().get(0); TableDesc tableDesc = PlanUtils.getIntermediateFileTableDesc(PlanUtils
Path emptyScratchDir = ctx.getMRTmpPath(); FileSystem fileSys = emptyScratchDir.getFileSystem(newJob); fileSys.mkdirs(emptyScratchDir); Utilities.setInputPaths(newJob, inputPaths); Utilities.setMapRedWork(newJob, selectTask.getWork(), ctx.getMRTmpPath());
continue; Path tmpPath = Utilities.generateTmpPath(ctx.getMRTmpPath(), originalTask.getId()); MapredLocalWork bigTableLocalWork = work.getMapRedLocalWork(); List<Operator<? extends OperatorDesc>> dummyOps =
continue; Path tmpPath = Utilities.generateTmpPath(ctx.getMRTmpPath(), originalTask.getId()); MapredLocalWork bigTableLocalWork = work.getMapRedLocalWork(); List<Operator<? extends OperatorDesc>> dummyOps =