@BeforeClass public static void setUpBeforeClass() throws Exception { TEST_UTIL.getConfiguration().setInt("hbase.regionserver.msginterval", 100); TEST_UTIL.getConfiguration().setInt("hbase.client.pause", 250); TEST_UTIL.getConfiguration().setInt("hbase.client.retries.number", 6); TEST_UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HIGH_PRIORITY_HANDLER_COUNT, 30); TEST_UTIL.getConfiguration().setInt(HConstants.REGION_SERVER_HANDLER_COUNT, 30); TEST_UTIL.getConfiguration().setBoolean("hbase.master.enabletable.roundrobin", true); TEST_UTIL.startMiniCluster(3); }
private static void clearOutput(Path path) throws IOException { FileSystem fs = path.getFileSystem(UTIL.getConfiguration()); if (fs.exists(path)) { assertEquals(true, fs.delete(path, true)); } }
private String setRootDirAndCleanIt(final HBaseTestingUtility htu, final String subdir) throws IOException { Path testdir = htu.getDataTestDir(subdir); FileSystem fs = FileSystem.get(htu.getConfiguration()); if (fs.exists(testdir)) assertTrue(fs.delete(testdir, true)); FSUtils.setRootDir(htu.getConfiguration(), testdir); return FSUtils.getRootDir(htu.getConfiguration()).toString(); }
/** * Assert that when a Coprocessor is annotated with CoreCoprocessor, then it is possible to * access a MasterServices instance. Assert the opposite too. * Do it to MasterCoprocessors. * @throws IOException */ @Test public void testCoreRegionCoprocessor() throws IOException { MasterCoprocessorEnvironment env = this.mch.load(null, NotCoreMasterCoprocessor.class.getName(), 0, HTU.getConfiguration()); assertFalse(env instanceof HasMasterServices); env = this.mch.load(null, CoreMasterCoprocessor.class.getName(), 1, HTU.getConfiguration()); assertTrue(env instanceof HasMasterServices); assertEquals(this.ms, ((HasMasterServices)env).getMasterServices()); } }
/** * Write to a log file with three concurrent threads and verifying all data is written. * @throws Exception */ @Test public void testConcurrentWrites() throws Exception { // Run the WPE tool with three threads writing 3000 edits each concurrently. // When done, verify that all edits were written. int errCode = WALPerformanceEvaluation. innerMain(new Configuration(TEST_UTIL.getConfiguration()), new String [] {"-threads", "3", "-verify", "-noclosefs", "-iterations", "3000"}); assertEquals(0, errCode); }
@Test public void testInvalidRegionServerHostnameAbortsServer() throws Exception { String invalidHostname = "hostAddr.invalid"; TEST_UTIL.getConfiguration().set(HRegionServer.RS_HOSTNAME_KEY, invalidHostname); HRegionServer hrs = null; try { hrs = new HRegionServer(TEST_UTIL.getConfiguration()); } catch (IllegalArgumentException iae) { assertTrue(iae.getMessage(), iae.getMessage().contains("Failed resolve of " + invalidHostname) || iae.getMessage().contains("Problem binding to " + invalidHostname)); } assertNull("Failed to validate against invalid hostname", hrs); }
@Test public void testCacheBlocks() throws IOException { // Set index block size to be the same as normal block size. TEST_UTIL.getConfiguration().setInt(HFileBlockIndex.MAX_CHUNK_SIZE_KEY, BLOCK_SIZE); BlockCache blockCache = BlockCacheFactory.createBlockCache(TEST_UTIL.getConfiguration()); ColumnFamilyDescriptor cfd = ColumnFamilyDescriptorBuilder.newBuilder(Bytes.toBytes(CF)).setMaxVersions(MAX_VERSIONS) .setCompressionType(COMPRESSION_ALGORITHM).setBloomFilterType(BLOOM_TYPE) .setBlocksize(BLOCK_SIZE).setBlockCacheEnabled(cfCacheEnabled).build(); HRegion region = TEST_UTIL.createTestRegion(TABLE, cfd, blockCache); CacheStats stats = blockCache.getStats(); writeTestData(region); assertEquals(0, stats.getHitCount()); assertEquals(0, HFile.DATABLOCK_READ_COUNT.sum()); // Do a single get, take count of caches. If we are NOT caching DATA blocks, the miss // count should go up. Otherwise, all should be cached and the miss count should not rise. region.get(new Get(Bytes.toBytes("row" + 0))); assertTrue(stats.getHitCount() > 0); assertTrue(HFile.DATABLOCK_READ_COUNT.sum() > 0); long missCount = stats.getMissCount(); region.get(new Get(Bytes.toBytes("row" + 0))); if (this.cfCacheEnabled) assertEquals(missCount, stats.getMissCount()); else assertTrue(stats.getMissCount() > missCount); }
@Test public void testReadingInvalidDirectoryFromFS() throws IOException { FileSystem fs = FileSystem.get(UTIL.getConfiguration()); try { new FSTableDescriptors(UTIL.getConfiguration(), fs, FSUtils.getRootDir(UTIL.getConfiguration())) .get(TableName.valueOf(HConstants.HBASE_TEMP_DIRECTORY)); fail("Shouldn't be able to read a table descriptor for the archive directory."); } catch (Exception e) { LOG.debug("Correctly got error when reading a table descriptor from the archive directory: " + e.getMessage()); } }
@Test public void testMiniClusterBindToWildcard() throws Exception { HBaseTestingUtility hbt = new HBaseTestingUtility(); hbt.getConfiguration().set("hbase.regionserver.ipc.address", "0.0.0.0"); MiniHBaseCluster cluster = hbt.startMiniCluster(); try { assertEquals(1, cluster.getLiveRegionServerThreads().size()); } finally { hbt.shutdownMiniCluster(); } }
/** * Run the RowCounter map reduce job and verify the row count. * * @param args the command line arguments to be used for rowcounter job. * @param expectedCount the expected row count (result of map reduce job). * @throws Exception */ private void runRowCount(String[] args, int expectedCount) throws Exception { Job job = RowCounter.createSubmittableJob(TEST_UTIL.getConfiguration(), args); long start = System.currentTimeMillis(); job.waitForCompletion(true); long duration = System.currentTimeMillis() - start; LOG.debug("row count duration (ms): " + duration); assertTrue(job.isSuccessful()); Counter counter = job.getCounters().findCounter(RowCounter.RowCounterMapper.Counters.ROWS); assertEquals(expectedCount, counter.getValue()); }
@Test(expected=RetriesExhaustedException.class) public void testSocketClosed() throws IOException, InterruptedException { TableName tableName = TableName.valueOf(name.getMethodName()); UTIL.createTable(tableName, fam1).close(); Configuration conf = new Configuration(UTIL.getConfiguration()); conf.set(RpcClientFactory.CUSTOM_RPC_CLIENT_IMPL_CONF_KEY, MyRpcClientImpl.class.getName()); conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 2); Connection connection = ConnectionFactory.createConnection(conf); Table table = connection.getTable(TableName.valueOf(name.getMethodName())); table.get(new Get(Bytes.toBytes("asd"))); connection.close(); for (Socket socket : MyRpcClientImpl.savedSockets) { assertTrue("Socket + " + socket + " is not closed", socket.isClosed()); } } }
@Test public void testThrottlingByMaxRitPercent() throws Exception { // Set max balancing time to 500 ms and max percent of regions in transition to 0.05 TEST_UTIL.getConfiguration().setInt(HConstants.HBASE_BALANCER_MAX_BALANCING, 500); TEST_UTIL.getConfiguration().setDouble(HConstants.HBASE_MASTER_BALANCER_MAX_RIT_PERCENT, 0.05); TEST_UTIL.startMiniCluster(2); TableName tableName = createTable("testThrottlingByMaxRitPercent"); final HMaster master = TEST_UTIL.getHBaseCluster().getMaster(); unbalance(master, tableName); AtomicInteger maxCount = new AtomicInteger(0); AtomicBoolean stop = new AtomicBoolean(false); Thread checker = startBalancerChecker(master, maxCount, stop); master.balance(); stop.set(true); checker.interrupt(); checker.join(); // The max number of regions in transition is 100 * 0.05 = 5 assertTrue("max regions in transition: " + maxCount.get(), maxCount.get() == 5); TEST_UTIL.deleteTable(tableName); }
@Test public void testMetrics() throws Exception { Configuration conf = UTIL.getConfiguration(); ThriftMetrics metrics = getMetrics(conf); ThriftHBaseServiceHandler hbaseHandler = createHandler(); THBaseService.Iface handler = HbaseHandlerMetricsProxy.newInstance(hbaseHandler, metrics, conf); byte[] rowName = Bytes.toBytes("testMetrics"); ByteBuffer table = wrap(tableAname); TGet get = new TGet(wrap(rowName)); assertFalse(handler.exists(table, get)); List<TColumnValue> columnValues = new ArrayList<>(2); columnValues.add(new TColumnValue(wrap(familyAname), wrap(qualifierAname), wrap(valueAname))); columnValues.add(new TColumnValue(wrap(familyBname), wrap(qualifierBname), wrap(valueBname))); TPut put = new TPut(wrap(rowName), columnValues); put.setColumnValues(columnValues); handler.put(table, put); assertTrue(handler.exists(table, get)); metricsHelper.assertCounter("put_num_ops", 1, metrics.getSource()); metricsHelper.assertCounter("exists_num_ops", 2, metrics.getSource()); }
@Test(expected = TableNotFoundException.class) public void testWithoutAnExistingTableAndCreateTableSetToNo() throws Exception { Configuration conf = util.getConfiguration(); conf.set(LoadIncrementalHFiles.CREATE_TABLE_CONF_KEY, "no"); LoadIncrementalHFiles loader = new LoadIncrementalHFiles(conf); String[] args = { "directory", "nonExistingTable" }; loader.run(args); }
private void testPriorityScenario(int expectedPriority, int limit, int stripes, int filesInStripe, int l0Files) throws Exception { final byte[][] keys = { KEY_A, KEY_B, KEY_C, KEY_D }; assertTrue(stripes <= keys.length + 1); Configuration conf = TEST_UTIL.getConfiguration(); conf.setInt("hbase.hstore.blockingStoreFiles", limit); StripeStoreFileManager sfm = createManager(al(), conf); for (int i = 0; i < l0Files; ++i) { sfm.insertNewFiles(al(createFile())); } for (int i = 0; i < filesInStripe; ++i) { ArrayList<HStoreFile> stripe = new ArrayList<>(); for (int j = 0; j < stripes; ++j) { stripe.add(createFile( (j == 0) ? OPEN_KEY : keys[j - 1], (j == stripes - 1) ? OPEN_KEY : keys[j])); } sfm.addCompactionResults(al(), stripe); } assertEquals(expectedPriority, sfm.getStoreCompactionPriority()); }
@Test public void testCombiner() throws IOException { Configuration conf = new Configuration(UTIL.getConfiguration()); // force use of combiner for testing purposes conf.setInt("mapreduce.map.combine.minspills", 1); runTestOnTable(UTIL.getConnection().getTable(MULTI_REGION_TABLE_NAME)); }
@Test public void testPutGetWithDelegationToken() throws Exception { TableName tableName = getTestTableName(); byte[] family = Bytes.toBytes("f"); byte[] qualifier = Bytes.toBytes("q"); byte[] row = Bytes.toBytes("row"); byte[] value = Bytes.toBytes("data"); try (Connection conn = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) { Admin admin = conn.getAdmin(); HTableDescriptor tableDescriptor = new HTableDescriptor(new HTableDescriptor(tableName)); tableDescriptor.addFamily(new HColumnDescriptor(family)); admin.createTable(tableDescriptor); try (Table table = conn.getTable(tableName)) { table.put(new Put(row).addColumn(family, qualifier, value)); Result result = table.get(new Get(row)); assertArrayEquals(value, result.getValue(family, qualifier)); } } } }
/** * Assert that when a Coprocessor is annotated with CoreCoprocessor, then it is possible to * access a RegionServerServices instance. Assert the opposite too. * Do it to RegionServerCoprocessors. * @throws IOException */ @Test public void testCoreRegionCoprocessor() throws IOException { RegionServerCoprocessorEnvironment env = rsch.load(null, NotCoreRegionServerCoprocessor.class.getName(), 0, HTU.getConfiguration()); assertFalse(env instanceof HasRegionServerServices); env = rsch.load(null, CoreRegionServerCoprocessor.class.getName(), 1, HTU.getConfiguration()); assertTrue(env instanceof HasRegionServerServices); assertEquals(this.rss, ((HasRegionServerServices)env).getRegionServerServices()); } }
@Test public void testFallbackToDefaultRpcControllerFactory() { Configuration conf = new Configuration(UTIL.getConfiguration()); conf.set(RpcControllerFactory.CUSTOM_CONTROLLER_CONF_KEY, "foo.bar.Baz"); // Should not fail RpcControllerFactory factory = RpcControllerFactory.instantiate(conf); assertNotNull(factory); assertEquals(factory.getClass(), RpcControllerFactory.class); } }
/** * Naive test to check that Connection#getAdmin returns a properly constructed HBaseAdmin object * @throws IOException Unable to construct admin */ @Test public void testAdminFactory() throws IOException { Connection con1 = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); Admin admin = con1.getAdmin(); assertTrue(admin.getConnection() == con1); assertTrue(admin.getConfiguration() == TEST_UTIL.getConfiguration()); con1.close(); }