private Path getArchiveDir() throws IOException { return new Path(UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY); }
private Path getTableDir(String tableName) throws IOException { Path testDataDir = UTIL.getDataTestDir(); FSUtils.setRootDir(UTIL.getConfiguration(), testDataDir); return new Path(testDataDir, tableName); }
/** * Create an HRegion that writes to the local tmp dirs. Creates the WAL for you. Be sure to call * {@link HBaseTestingUtility#closeRegionAndWAL(HRegion)} when you're finished with it. */ public HRegion createLocalHRegion(RegionInfo info, TableDescriptor desc) throws IOException { return createRegionAndWAL(info, getDataTestDir(), getConfiguration(), desc); }
@Test public void testNoSuchTable() throws IOException { final String name = "testNoSuchTable"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any detrius laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); assertNull("There shouldn't be any HTD for this table", htds.get(TableName.valueOf("NoSuchTable"))); }
@Ignore("Goes zombie too frequently; needs work. See HBASE-14563") @Test public void testJobConfiguration() throws Exception { Configuration conf = new Configuration(this.util.getConfiguration()); conf.set(HConstants.TEMPORARY_FS_DIRECTORY_KEY, util.getDataTestDir("testJobConfiguration") .toString()); Job job = new Job(conf); job.setWorkingDirectory(util.getDataTestDir("testJobConfiguration")); Table table = Mockito.mock(Table.class); RegionLocator regionLocator = Mockito.mock(RegionLocator.class); setupMockStartKeys(regionLocator); setupMockTableName(regionLocator); HFileOutputFormat2.configureIncrementalLoad(job, table.getTableDescriptor(), regionLocator); assertEquals(job.getNumReduceTasks(), 4); }
@Test public void testFileBucketCacheConfig() throws IOException { HBaseTestingUtility htu = new HBaseTestingUtility(this.conf); try { Path p = new Path(htu.getDataTestDir(), "bc.txt"); FileSystem fs = FileSystem.get(this.conf); fs.create(p).close(); this.conf.set(HConstants.BUCKET_CACHE_IOENGINE_KEY, "file:" + p); doBucketCacheConfigTest(); } finally { htu.cleanupTestDir(); } }
/** * Test, on a local filesystem, that the FileLink is still readable * even when the current file gets renamed. */ @Test public void testLocalLinkReadDuringRename() throws IOException { HBaseTestingUtility testUtil = new HBaseTestingUtility(); FileSystem fs = testUtil.getTestFileSystem(); assertEquals("file", fs.getUri().getScheme()); testLinkReadDuringRename(fs, testUtil.getDataTestDir()); }
@Test public void test() throws IOException, InterruptedException, ExecutionException, FSUtils.StreamLacksCapabilityException { Path f = new Path(TEST_UTIL.getDataTestDir(), "test"); FileSystem fs = FileSystem.getLocal(TEST_UTIL.getConfiguration()); AsyncFSOutput out = AsyncFSOutputHelper.createOutput(fs, f, false, true, fs.getDefaultReplication(f), fs.getDefaultBlockSize(f), GROUP, CHANNEL_CLASS); TestFanOutOneBlockAsyncDFSOutput.writeAndVerify(fs, f, out); } }
@Test public void testReadingHTDFromFS() throws IOException { final String name = this.name.getMethodName(); FileSystem fs = FileSystem.get(UTIL.getConfiguration()); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(); Path rootdir = UTIL.getDataTestDir(name); FSTableDescriptors fstd = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); fstd.createTableDescriptor(htd); TableDescriptor td2 = FSTableDescriptors.getTableDescriptorFromFs(fs, rootdir, htd.getTableName()); assertTrue(htd.equals(td2)); }
@Test public void testDeleteAndExists() throws Exception { final Path rootdir = htu.getDataTestDir(); final FileSystem fs = rootdir.getFileSystem(conf); conf.setBoolean(HConstants.ENABLE_DATA_FILE_UMASK, true); FsPermission perms = FSUtils.getFilePermissions(fs, conf, HConstants.DATA_FILE_UMASK_KEY); // then that the correct file is created String file = htu.getRandomUUID().toString(); Path p = new Path(htu.getDataTestDir(), "temptarget" + File.separator + file); Path p1 = new Path(htu.getDataTestDir(), "temppath" + File.separator + file); try { FSDataOutputStream out = FSUtils.create(conf, fs, p, perms, null); out.close(); assertTrue("The created file should be present", FSUtils.isExists(fs, p)); // delete the file with recursion as false. Only the file will be deleted. FSUtils.delete(fs, p, false); // Create another file FSDataOutputStream out1 = FSUtils.create(conf, fs, p1, perms, null); out1.close(); // delete the file with recursion as false. Still the file only will be deleted FSUtils.delete(fs, p1, true); assertFalse("The created file should be present", FSUtils.isExists(fs, p1)); // and then cleanup } finally { FSUtils.delete(fs, p, true); FSUtils.delete(fs, p1, true); } }
@Test public void testTestDir() throws Exception { HBaseTestingUtility hbt = new HBaseTestingUtility(); Path testdir = hbt.getDataTestDir(); LOG.info("testdir=" + testdir); FileSystem fs = hbt.getTestFileSystem(); assertTrue(!fs.exists(testdir)); assertTrue(fs.mkdirs(testdir)); assertTrue(hbt.cleanupTestDir()); }
private String getNotExistFilePath() { Path path = new Path(TEST_UTIL.getDataTestDir(), "does_not_exist"); return path.toUri().getPath(); }
@Before public void setup() throws Exception { rootDir = TEST_UTIL.getDataTestDir("testRestore"); archiveDir = new Path(rootDir, HConstants.HFILE_ARCHIVE_DIRECTORY); fs = TEST_UTIL.getTestFileSystem(); conf = TEST_UTIL.getConfiguration(); setupConf(conf); FSUtils.setRootDir(conf, rootDir); }
/** * Create an HRegion that writes to the local tmp dirs with specified wal * @param info regioninfo * @param desc table descriptor * @param wal wal for this region. * @return created hregion * @throws IOException */ public HRegion createLocalHRegion(RegionInfo info, TableDescriptor desc, WAL wal) throws IOException { return HRegion.createHRegion(info, getDataTestDir(), getConfiguration(), desc, wal); }
@Test public void testUpdates() throws IOException { final String name = "testUpdates"; FileSystem fs = FileSystem.get(UTIL.getConfiguration()); // Cleanup old tests if any detrius laying around. Path rootdir = new Path(UTIL.getDataTestDir(), name); TableDescriptors htds = new FSTableDescriptors(UTIL.getConfiguration(), fs, rootdir); TableDescriptor htd = TableDescriptorBuilder.newBuilder(TableName.valueOf(name)).build(); htds.add(htd); htds.add(htd); htds.add(htd); }
@Test public void testMinimumNumberOfThreads() throws Exception { Stoppable stop = new StoppableImplementation(); Configuration conf = UTIL.getConfiguration(); Path testDir = UTIL.getDataTestDir(); FileSystem fs = UTIL.getTestFileSystem(); String confKey = "hbase.test.cleaner.delegates"; conf.set(confKey, AlwaysDelete.class.getName()); conf.set(CleanerChore.CHORE_POOL_SIZE, "2"); AllValidPaths chore = new AllValidPaths("test-file-cleaner", stop, conf, fs, testDir, confKey); int numProcs = Runtime.getRuntime().availableProcessors(); // Sanity assertEquals(numProcs, chore.calculatePoolSize(Integer.toString(numProcs))); // The implementation does not allow us to set more threads than we have processors assertEquals(numProcs, chore.calculatePoolSize(Integer.toString(numProcs + 2))); // Force us into the branch that is multiplying 0.0 against the number of processors assertEquals(1, chore.calculatePoolSize("0.0")); }
@Test public void testDataBlockEncryption() throws IOException { final int blocks = 10; Path path = new Path(TEST_UTIL.getDataTestDir(), "block_v3_" + compression + "_AES"); LOG.info("testDataBlockEncryption: encryption=AES compression=" + compression); long totalSize = 0;
@Test public void testShouldFlushMeta() throws Exception { // write an edit in the META and ensure the shouldFlush (that the periodic memstore // flusher invokes) returns true after SYSTEM_CACHE_FLUSH_INTERVAL (even though // the MEMSTORE_PERIODIC_FLUSH_INTERVAL is set to a higher value) Configuration conf = new Configuration(); conf.setInt(HRegion.MEMSTORE_PERIODIC_FLUSH_INTERVAL, HRegion.SYSTEM_CACHE_FLUSH_INTERVAL * 10); HBaseTestingUtility hbaseUtility = HBaseTestingUtility.createLocalHTU(conf); Path testDir = hbaseUtility.getDataTestDir(); EnvironmentEdgeForMemstoreTest edge = new EnvironmentEdgeForMemstoreTest(); EnvironmentEdgeManager.injectEdge(edge); edge.setCurrentTimeMillis(1234); WALFactory wFactory = new WALFactory(conf, "1234"); HRegion meta = HRegion.createHRegion(RegionInfoBuilder.FIRST_META_REGIONINFO, testDir, conf, FSTableDescriptors.createMetaTableDescriptor(conf), wFactory.getWAL(RegionInfoBuilder.FIRST_META_REGIONINFO)); // parameterized tests add [#] suffix get rid of [ and ]. TableDescriptor desc = TableDescriptorBuilder .newBuilder(TableName.valueOf(name.getMethodName().replaceAll("[\\[\\]]", "_"))) .setColumnFamily(ColumnFamilyDescriptorBuilder.of("foo")).build(); RegionInfo hri = RegionInfoBuilder.newBuilder(desc.getTableName()) .setStartKey(Bytes.toBytes("row_0200")).setEndKey(Bytes.toBytes("row_0300")).build(); HRegion r = HRegion.createHRegion(hri, testDir, conf, desc, wFactory.getWAL(hri)); addRegionToMETA(meta, r); edge.setCurrentTimeMillis(1234 + 100); StringBuilder sb = new StringBuilder(); assertTrue(meta.shouldFlush(sb) == false); edge.setCurrentTimeMillis(edge.currentTime() + HRegion.SYSTEM_CACHE_FLUSH_INTERVAL + 1); assertTrue(meta.shouldFlush(sb) == true); }
private void testHFileFormatV3Internals(boolean useTags) throws IOException { Path hfilePath = new Path(TEST_UTIL.getDataTestDir(), "testHFileFormatV3"); final Compression.Algorithm compressAlgo = Compression.Algorithm.GZ; final int entryCount = 10000; writeDataAndReadFromHFile(hfilePath, compressAlgo, entryCount, false, useTags); }
Path makeNewFile(TagUsage tagUsage) throws IOException { Path ncTFile = new Path(TEST_UTIL.getDataTestDir(), "basic.hfile"); FSDataOutputStream fout = TEST_UTIL.getTestFileSystem().create(ncTFile); int blocksize = toKV("a", tagUsage).getLength() * 3; HFileContext context = new HFileContextBuilder().withBlockSize(blocksize) .withDataBlockEncoding(encoding) .withIncludesTags(true).build(); Configuration conf = TEST_UTIL.getConfiguration(); HFile.Writer writer = HFile.getWriterFactoryNoCache(conf).withOutputStream(fout) .withFileContext(context) .withComparator(CellComparatorImpl.COMPARATOR).create(); // 4 bytes * 3 * 2 for each key/value + // 3 for keys, 15 for values = 42 (woot) writer.append(toKV("c", tagUsage)); writer.append(toKV("e", tagUsage)); writer.append(toKV("g", tagUsage)); // block transition writer.append(toKV("i", tagUsage)); writer.append(toKV("k", tagUsage)); writer.close(); fout.close(); return ncTFile; }