public boolean deleteOnExit(Path f) throws java.io.IOException { return this.underlyingFs.deleteOnExit(replaceScheme(f, this.replacementScheme, this.underlyingScheme)); }
public static Path getBulkOutputDir(String tableName, Configuration conf, boolean deleteOnExit) throws IOException { FileSystem fs = FileSystem.get(conf); String tmp = conf.get(HConstants.TEMPORARY_FS_DIRECTORY_KEY, fs.getHomeDirectory() + "/hbase-staging"); Path path = new Path(tmp + Path.SEPARATOR + "bulk_output-" + tableName + "-" + EnvironmentEdgeManager.currentTime()); if (deleteOnExit) { fs.deleteOnExit(path); } return path; }
private Path createScratchDir() throws IOException { Path parent = new Path(SessionState.get().getHdfsScratchDirURIString(), SPARK_DIR); Path sparkDir = new Path(parent, sessionId); FileSystem fs = sparkDir.getFileSystem(conf); FsPermission fsPermission = new FsPermission(HiveConf.getVar( conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION)); fs.mkdirs(sparkDir, fsPermission); fs.deleteOnExit(sparkDir); return sparkDir; }
private void deleteDirOnExit(FileSystem fs, String target) throws Exception { Path path = new Path(extractPathFromUrl(target)); fs.deleteOnExit(path); // Delete the directory even if an exception occurs info(path + " is scheduled to be deleted on exit."); }
private Path createScratchDir() throws IOException { Path parent = new Path(SessionState.get().getHdfsScratchDirURIString(), SPARK_DIR); Path sparkDir = new Path(parent, sessionId); FileSystem fs = sparkDir.getFileSystem(conf); FsPermission fsPermission = new FsPermission(HiveConf.getVar( conf, HiveConf.ConfVars.SCRATCHDIRPERMISSION)); fs.mkdirs(sparkDir, fsPermission); fs.deleteOnExit(sparkDir); return sparkDir; }
/** * Configure <code>job</code> with a TotalOrderPartitioner, partitioning against * <code>splitPoints</code>. Cleans up the partitions file after job exists. */ static void configurePartitioner(Job job, List<ImmutableBytesWritable> splitPoints) throws IOException { Configuration conf = job.getConfiguration(); // create the partitions file FileSystem fs = FileSystem.get(conf); Path partitionsPath = new Path(conf.get("hbase.fs.tmp.dir"), "partitions_" + RandomUtil.randomUUID()); fs.makeQualified(partitionsPath); writePartitions(conf, partitionsPath, splitPoints); fs.deleteOnExit(partitionsPath); // configure job to use it job.setPartitionerClass(TotalOrderPartitioner.class); TotalOrderPartitioner.setPartitionFile(conf, partitionsPath); }
private QueryResultsCache(HiveConf configuration) throws IOException { this.conf = configuration; // Set up cache directory Path rootCacheDir = new Path(conf.getVar(HiveConf.ConfVars.HIVE_QUERY_RESULTS_CACHE_DIRECTORY)); LOG.info("Initializing query results cache at {}", rootCacheDir); Utilities.ensurePathIsWritable(rootCacheDir, conf); String currentCacheDirName = "results-" + UUID.randomUUID().toString(); cacheDirPath = new Path(rootCacheDir, currentCacheDirName); FileSystem fs = cacheDirPath.getFileSystem(conf); FsPermission fsPermission = new FsPermission("700"); fs.mkdirs(cacheDirPath, fsPermission); // Create non-existent path for 0-row results zeroRowsPath = new Path(cacheDirPath, "dummy_zero_rows"); // Results cache directory should be cleaned up at process termination. fs.deleteOnExit(cacheDirPath); maxCacheSize = conf.getLongVar(HiveConf.ConfVars.HIVE_QUERY_RESULTS_CACHE_MAX_SIZE); maxEntrySize = conf.getLongVar(HiveConf.ConfVars.HIVE_QUERY_RESULTS_CACHE_MAX_ENTRY_SIZE); maxEntryLifetime = conf.getTimeVar( HiveConf.ConfVars.HIVE_QUERY_RESULTS_CACHE_MAX_ENTRY_LIFETIME, TimeUnit.MILLISECONDS); LOG.info("Query results cache: cacheDirectory {}, maxCacheSize {}, maxEntrySize {}, maxEntryLifetime {}", cacheDirPath, maxCacheSize, maxEntrySize, maxEntryLifetime); }
/** * Configure <code>job</code> with a TotalOrderPartitioner, partitioning against * <code>splitPoints</code>. Cleans up the partitions file after job exists. */ static void configurePartitioner(Job job, List<ImmutableBytesWritable> splitPoints, boolean writeMultipleTables) throws IOException { Configuration conf = job.getConfiguration(); // create the partitions file FileSystem fs = FileSystem.get(conf); String hbaseTmpFsDir = conf.get(HConstants.TEMPORARY_FS_DIRECTORY_KEY, fs.getHomeDirectory() + "/hbase-staging"); Path partitionsPath = new Path(hbaseTmpFsDir, "partitions_" + UUID.randomUUID()); fs.makeQualified(partitionsPath); writePartitions(conf, partitionsPath, splitPoints, writeMultipleTables); fs.deleteOnExit(partitionsPath); // configure job to use it job.setPartitionerClass(TotalOrderPartitioner.class); TotalOrderPartitioner.setPartitionFile(conf, partitionsPath); }
private void createFile(String partPath, String filename) throws IOException { Path part = new Path(partPath); fs.mkdirs(part); fs.createNewFile(new Path(partPath + Path.SEPARATOR + filename)); fs.deleteOnExit(part); }
public Path getTestDir(String testName, String subdir) throws IOException { Path testDir = util.getDataTestDirOnTestFS(testName); FileSystem fs = FileSystem.get(getConf()); fs.deleteOnExit(testDir); return new Path(new Path(testDir, testName), subdir); }
@Override public void configure(JobConf job) { jc = job; work = (ColumnTruncateWork) Utilities.getMapWork(job); Path specPath = work.getOutputDir(); Path tmpPath = Utilities.toTempPath(specPath); Path taskTmpPath = Utilities.toTaskTempPath(specPath); updatePaths(tmpPath, taskTmpPath); try { fs = specPath.getFileSystem(job); autoDelete = fs.deleteOnExit(outPath); } catch (IOException e) { this.exception = true; throw new RuntimeException(e); } }
private Path createFile(FileSystem fs, FsPermission perms) throws IOException { StringBuilder buf = new StringBuilder(); for (int i = 0; i < 10; i++) { buf.append((char)(rand.nextInt(26) + 'a')); } Path p = new Path(buf.toString()); FSDataOutputStream os = fs.create(p); os.writeBytes("Mary had a little lamb\nit's fleece was white as snow\nand anywhere that Mary " + "went\nthe lamb was sure to go\n"); os.close(); fs.setPermission(p, perms); fs.deleteOnExit(p); return p; }
private void createDirectory(String partPath) throws IOException { Path part = new Path(partPath); fs.mkdirs(part); // create files under partitions to simulate real partitions fs.createNewFile(new Path(partPath + Path.SEPARATOR + "dummydata1")); fs.createNewFile(new Path(partPath + Path.SEPARATOR + "dummydata2")); fs.deleteOnExit(part); } }
@Override public void configure(JobConf job) { jc = job; work = (ColumnTruncateWork) Utilities.getMapWork(job); Path specPath = work.getOutputDir(); Path tmpPath = Utilities.toTempPath(specPath); Path taskTmpPath = Utilities.toTaskTempPath(specPath); updatePaths(tmpPath, taskTmpPath); try { fs = specPath.getFileSystem(job); autoDelete = fs.deleteOnExit(outPath); } catch (IOException e) { this.exception = true; throw new RuntimeException(e); } }
/** * Sets up a new path in test filesystem to be used by tests. */ private Path getNewDataTestDirOnTestFS() throws IOException { //The file system can be either local, mini dfs, or if the configuration //is supplied externally, it can be an external cluster FS. If it is a local //file system, the tests should use getBaseTestDir, otherwise, we can use //the working directory, and create a unique sub dir there FileSystem fs = getTestFileSystem(); Path newDataTestDir; String randomStr = getRandomUUID().toString(); if (fs.getUri().getScheme().equals(FileSystem.getLocal(conf).getUri().getScheme())) { newDataTestDir = new Path(getDataTestDir(), randomStr); File dataTestDir = new File(newDataTestDir.toString()); if (deleteOnExit()) dataTestDir.deleteOnExit(); } else { Path base = getBaseTestDirOnTestFS(); newDataTestDir = new Path(base, randomStr); if (deleteOnExit()) fs.deleteOnExit(newDataTestDir); } return newDataTestDir; }
@Test(expected = MetastoreException.class) public void testInvalidPartitionKeyName() throws HiveException, AlreadyExistsException, IOException, MetastoreException { Table table = createTestTable(); List<Partition> partitions = hive.getPartitions(table); assertEquals(2, partitions.size()); // add a fake partition dir on fs fs = partitions.get(0).getDataLocation().getFileSystem(hive.getConf()); Path fakePart = new Path(table.getDataLocation().toString(), "fakedate=2009-01-01/fakecity=sanjose"); fs.mkdirs(fakePart); fs.deleteOnExit(fakePart); checker.checkMetastore(catName, dbName, tableName, null, new CheckResult()); }
@Test public void TestGetJobWorkingDir() throws IOException { FileSystem fileSystem = FileSystem.get(new Configuration()); Path jobWorkDirPath = null; KylinConfig kylinConfig = mock(KylinConfig.class); try (SetAndUnsetThreadLocalConfig autoUnset = KylinConfig.setAndUnsetThreadLocalConfig(kylinConfig)) { when(kylinConfig.getHiveTableDirCreateFirst()).thenReturn(true); when(kylinConfig.getHdfsWorkingDirectory()).thenReturn("/tmp/kylin/"); DefaultChainedExecutable defaultChainedExecutable = mock(DefaultChainedExecutable.class); defaultChainedExecutable.setId(RandomUtil.randomUUID().toString()); String jobWorkingDir = HiveInputBase.getJobWorkingDir(defaultChainedExecutable, KylinConfig.getInstanceFromEnv().getHdfsWorkingDirectory()); jobWorkDirPath = new Path(jobWorkingDir); Assert.assertTrue(fileSystem.exists(jobWorkDirPath)); } finally { if (jobWorkDirPath != null) fileSystem.deleteOnExit(jobWorkDirPath); } }
@Test public void testAdditionalPartitionDirs() throws HiveException, AlreadyExistsException, IOException, MetastoreException { Table table = createTestTable(); List<Partition> partitions = hive.getPartitions(table); assertEquals(2, partitions.size()); // add a fake partition dir on fs fs = partitions.get(0).getDataLocation().getFileSystem(hive.getConf()); Path fakePart = new Path(table.getDataLocation().toString(), partDateName + "=2017-01-01/" + partCityName + "=paloalto/fakePartCol=fakepartValue"); fs.mkdirs(fakePart); fs.deleteOnExit(fakePart); CheckResult result = new CheckResult(); checker.checkMetastore(catName, dbName, tableName, null, result); assertEquals(Collections.<String> emptySet(), result.getTablesNotInMs()); assertEquals(Collections.<String> emptySet(), result.getTablesNotOnFs()); assertEquals(Collections.<CheckResult.PartitionResult> emptySet(), result.getPartitionsNotOnFs()); //fakePart path partition is added since the defined partition keys are valid assertEquals(1, result.getPartitionsNotInMs().size()); }
@Test public void testDataDeletion() throws HiveException, IOException, TException { Database db = new Database(); db.setName(dbName); hive.createDatabase(db); Table table = new Table(dbName, tableName); table.setDbName(dbName); table.setInputFormatClass(TextInputFormat.class); table.setOutputFormatClass(HiveIgnoreKeyTextOutputFormat.class); table.setPartCols(partCols); hive.createTable(table); table = hive.getTable(dbName, tableName); Path fakeTable = table.getPath().getParent().suffix( Path.SEPARATOR + "faketable"); fs = fakeTable.getFileSystem(hive.getConf()); fs.mkdirs(fakeTable); fs.deleteOnExit(fakeTable); Path fakePart = new Path(table.getDataLocation().toString(), "fakepartition=fakevalue"); fs.mkdirs(fakePart); fs.deleteOnExit(fakePart); hive.dropTable(dbName, tableName, true, true); assertFalse(fs.exists(fakePart)); hive.dropDatabase(dbName); assertFalse(fs.exists(fakeTable)); }
@Test public void testSkipInvalidPartitionKeyName() throws HiveException, AlreadyExistsException, IOException, MetastoreException { hive.getConf().set(HiveConf.ConfVars.HIVE_MSCK_PATH_VALIDATION.varname, "skip"); checker = new HiveMetaStoreChecker(msc, hive.getConf()); Table table = createTestTable(); List<Partition> partitions = hive.getPartitions(table); assertEquals(2, partitions.size()); // add a fake partition dir on fs fs = partitions.get(0).getDataLocation().getFileSystem(hive.getConf()); Path fakePart = new Path(table.getDataLocation().toString(), "fakedate=2009-01-01/fakecity=sanjose"); fs.mkdirs(fakePart); fs.deleteOnExit(fakePart); createPartitionsDirectoriesOnFS(table, 2); CheckResult result = new CheckResult(); checker.checkMetastore(catName, dbName, tableName, null, result); assertEquals(Collections.<String> emptySet(), result.getTablesNotInMs()); assertEquals(Collections.<String> emptySet(), result.getTablesNotOnFs()); assertEquals(Collections.<CheckResult.PartitionResult> emptySet(), result.getPartitionsNotOnFs()); // only 2 valid partitions should be added assertEquals(2, result.getPartitionsNotInMs().size()); }