private void cleanup(Admin admin, Deque<LoadQueueItem> queue, ExecutorService pool, SecureBulkLoadClient secureClient) throws IOException { fsDelegationToken.releaseDelegationToken(); if (bulkToken != null && secureClient != null) { secureClient.cleanupBulkLoad(admin.getConnection(), bulkToken); } if (pool != null) { pool.shutdown(); } if (!queue.isEmpty()) { StringBuilder err = new StringBuilder(); err.append("-------------------------------------------------\n"); err.append("Bulk load aborted with some files not yet loaded:\n"); err.append("-------------------------------------------------\n"); for (LoadQueueItem q : queue) { err.append(" ").append(q.getFilePath()).append('\n'); } LOG.error(err.toString()); } }
@Override public void close() throws IOException { if (admin != null) { Connection connection = admin.getConnection(); admin.close(); admin = null; if (connection != null) { connection.close(); } } } }
private static List<Future<Void>> sniff(final Admin admin, final Sink sink, TableDescriptor tableDesc, ExecutorService executor, TaskType taskType, boolean rawScanEnabled, LongAdder rwLatency) throws Exception { LOG.debug("Reading list of regions for table {}", tableDesc.getTableName()); try (Table table = admin.getConnection().getTable(tableDesc.getTableName())) { List<RegionTask> tasks = new ArrayList<>(); try (RegionLocator regionLocator = admin.getConnection().getRegionLocator(tableDesc.getTableName())) { for (HRegionLocation location: regionLocator.getAllRegionLocations()) { ServerName rs = location.getServerName(); RegionInfo region = location.getRegion(); tasks.add(new RegionTask(admin.getConnection(), region, rs, (RegionStdOutSink)sink, taskType, rawScanEnabled, rwLatency)); } return executor.invokeAll(tasks); } } catch (TableNotFoundException e) { return Collections.EMPTY_LIST; } }
for (TableDescriptor tableDesc: tableDescs) { try (RegionLocator regionLocator = this.admin.getConnection().getRegionLocator(tableDesc.getTableName())) { for (HRegionLocation location : regionLocator.getAllRegionLocations()) { ServerName rs = location.getServerName();
@Test public void testCreateTableWithSingleReplica() throws Exception { final int numRegions = 3; final int numReplica = 1; final TableName tableName = TableName.valueOf(name.getMethodName()); try { HTableDescriptor desc = new HTableDescriptor(tableName); desc.setRegionReplication(numReplica); desc.addFamily(new HColumnDescriptor("family")); ADMIN.createTable(desc, Bytes.toBytes("A"), Bytes.toBytes("Z"), numRegions); validateNumberOfRowsInMeta(tableName, numRegions, ADMIN.getConnection()); List<RegionInfo> hris = MetaTableAccessor.getTableRegions( ADMIN.getConnection(), tableName); assert(hris.size() == numRegions * numReplica); } finally { ADMIN.disableTable(tableName); ADMIN.deleteTable(tableName); } }
@Test public void testSplitShouldNotHappenIfSplitIsDisabledForTable() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); HTableDescriptor htd = new HTableDescriptor(tableName); htd.addFamily(new HColumnDescriptor("f")); htd.setRegionSplitPolicyClassName(DisabledRegionSplitPolicy.class.getName()); Table table = TEST_UTIL.createTable(htd, null); for(int i = 0; i < 10; i++) { Put p = new Put(Bytes.toBytes("row"+i)); byte[] q1 = Bytes.toBytes("q1"); byte[] v1 = Bytes.toBytes("v1"); p.addColumn(Bytes.toBytes("f"), q1, v1); table.put(p); } this.admin.flush(tableName); try { this.admin.split(tableName, Bytes.toBytes("row5")); Threads.sleep(10000); } catch (Exception e) { // Nothing to do. } // Split should not happen. List<RegionInfo> allRegions = MetaTableAccessor.getTableRegions( this.admin.getConnection(), tableName, true); assertEquals(1, allRegions.size()); }
@Test public void testRoundRobinAssignment() throws Exception { TableName tableName = TableName.valueOf("testRoundRobinAssignment"); HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM); TEST_UTIL.waitTableAvailable(tableName); TEST_UTIL.loadTable(admin.getConnection().getTable(tableName), HConstants.CATALOG_FAMILY); admin.flush(tableName); LoadBalancer balancer = master.getLoadBalancer(); List<RegionInfo> regions = admin.getRegions(tableName); regions.addAll(admin.getTableRegions(TableName.META_TABLE_NAME)); List<ServerName> servers = Lists.newArrayList( admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)).getLiveServerMetrics().keySet()); Map<ServerName, List<RegionInfo>> map = balancer.roundRobinAssignment(regions, servers); for (List<RegionInfo> regionInfos : map.values()) { regions.removeAll(regionInfos); } assertEquals("No region should be missed by balancer", 0, regions.size()); }
@Test public void testRegionCache() throws IOException { HTableDescriptor htd = new HTableDescriptor(TableName.valueOf(name.getMethodName())); HColumnDescriptor fam = new HColumnDescriptor(FAMILY); htd.addFamily(fam); byte[][] KEYS = HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE; Admin admin = TEST_UTIL.getAdmin(); admin.createTable(htd, KEYS); HRegionLocator locator = (HRegionLocator) admin.getConnection().getRegionLocator(htd.getTableName()); List<HRegionLocation> results = locator.getAllRegionLocations(); int number = ((ConnectionImplementation)admin.getConnection()) .getNumberOfCachedRegionLocations(htd.getTableName()); assertEquals(results.size(), number); ConnectionImplementation conn = ((ConnectionImplementation)admin.getConnection()); assertNotNull("Can't get cached location for row aaa", conn.getCachedLocation(htd.getTableName(),Bytes.toBytes("aaa"))); for(byte[] startKey:HBaseTestingUtility.KEYS_FOR_HBA_CREATE_TABLE){ assertNotNull("Can't get cached location for row "+ Bytes.toString(startKey),(conn.getCachedLocation(htd.getTableName(),startKey))); } }
@Before public void before() throws IOException { ((ClusterConnection) HTU.getAdmin().getConnection()).clearRegionCache(); try { openRegion(hriPrimary); } catch (Exception ignored) { } try { openRegion(hriSecondary); } catch (Exception ignored) { } }
@After public void after() throws IOException, KeeperException { try { closeRegion(hriSecondary); } catch (Exception ignored) { } try { closeRegion(hriPrimary); } catch (Exception ignored) { } ((ClusterConnection) HTU.getAdmin().getConnection()).clearRegionCache(); }
try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(tableName)) { tableLength = locator.getStartKeys().length; try (RegionLocator locator = ADMIN.getConnection().getRegionLocator(cloneTableName)) { tableLength = locator.getStartKeys().length;
(HRegionLocator) admin.getConnection().getRegionLocator(htd.getTableName()); for (int regionReplication = 1; regionReplication < 4; regionReplication++) { List<RegionLocations> regionLocations = new ArrayList<>();
Assert.assertEquals(expectedStoreFilesSize, store.getSize()); ClusterConnection conn = ((ClusterConnection) admin.getConnection()); HBaseRpcController controller = conn.getRpcControllerFactory().newController(); for (int i = 0; i < 10; i++) {
Connection conn = admin.getConnection(); final byte[] rowKey = Bytes.toBytes("row1"); final byte[] col = Bytes.toBytes("q1");
Connection conn = admin.getConnection(); final byte[] rowKey = Bytes.toBytes("row1"); final byte[] col = Bytes.toBytes("q1");
/** * Naive test to check that Connection#getAdmin returns a properly constructed HBaseAdmin object * @throws IOException Unable to construct admin */ @Test public void testAdminFactory() throws IOException { Connection con1 = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration()); Admin admin = con1.getAdmin(); assertTrue(admin.getConnection() == con1); assertTrue(admin.getConfiguration() == TEST_UTIL.getConfiguration()); con1.close(); }
@Test public void testBasicBalance() throws Exception { TableName tableName = TableName.valueOf("testBasicBalance"); HTableDescriptor desc = new HTableDescriptor(tableName); desc.addFamily(new HColumnDescriptor(HConstants.CATALOG_FAMILY)); admin.createTable(desc, Bytes.toBytes("aaa"), Bytes.toBytes("zzz"), REGION_NUM); TEST_UTIL.waitTableAvailable(tableName); TEST_UTIL.loadTable(admin.getConnection().getTable(tableName), HConstants.CATALOG_FAMILY); admin.flush(tableName); compactTable(tableName); JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServerAndWait(10000); JVMClusterUtil.RegionServerThread rs2 = cluster.startRegionServerAndWait(10000); // Now try to run balance, and verify no regions are moved to the 2 region servers recently // started. admin.setBalancerRunning(true, true); assertTrue("Balancer did not run", admin.balancer()); TEST_UTIL.waitUntilNoRegionsInTransition(120000); List<RegionInfo> hris = admin.getRegions(rs1.getRegionServer().getServerName()); for (RegionInfo hri : hris) { assertFalse("New RS contains regions belonging to table: " + tableName, hri.getTable().equals(tableName)); } hris = admin.getRegions(rs2.getRegionServer().getServerName()); for (RegionInfo hri : hris) { assertFalse("New RS contains regions belonging to table: " + tableName, hri.getTable().equals(tableName)); } }
@Test public void testGetRegionLocationFromPrimaryMetaRegion() throws IOException, InterruptedException { HTU.getAdmin().setBalancerRunning(false, true); ((ConnectionImplementation) HTU.getAdmin().getConnection()).setUseMetaReplicas(true); // Create table then get the single region for our new table. HTableDescriptor hdt = HTU.createTableDescriptor("testGetRegionLocationFromPrimaryMetaRegion"); hdt.setRegionReplication(2); try { HTU.createTable(hdt, new byte[][] { f }, null); RegionServerHostingPrimayMetaRegionSlowOrStopCopro.slowDownPrimaryMetaScan = true; // Get user table location, always get it from the primary meta replica RegionLocations url = ((ClusterConnection) HTU.getConnection()) .locateRegion(hdt.getTableName(), row, false, false); } finally { RegionServerHostingPrimayMetaRegionSlowOrStopCopro.slowDownPrimaryMetaScan = false; ((ConnectionImplementation) HTU.getAdmin().getConnection()).setUseMetaReplicas(false); HTU.getAdmin().setBalancerRunning(true, true); HTU.getAdmin().disableTable(hdt.getTableName()); HTU.deleteTable(hdt.getTableName()); } }
@Test @SuppressWarnings("deprecation") public void testShortCircuitConnection() throws IOException, InterruptedException { final TableName tableName = TableName.valueOf(name.getMethodName()); HTableDescriptor htd = UTIL.createTableDescriptor(tableName); HColumnDescriptor hcd = new HColumnDescriptor(Bytes.toBytes("cf")); htd.addFamily(hcd); UTIL.createTable(htd, null); HRegionServer regionServer = UTIL.getRSForFirstRegionInTable(tableName); ClusterConnection connection = regionServer.getClusterConnection(); Table tableIf = connection.getTable(tableName); assertTrue(tableIf instanceof HTable); HTable table = (HTable) tableIf; assertTrue(table.getConnection() == connection); AdminService.BlockingInterface admin = connection.getAdmin(regionServer.getServerName()); ClientService.BlockingInterface client = connection.getClient(regionServer.getServerName()); assertTrue(admin instanceof RSRpcServices); assertTrue(client instanceof RSRpcServices); ServerName anotherSn = ServerName.valueOf(regionServer.getServerName().getHostAndPort(), EnvironmentEdgeManager.currentTime()); admin = connection.getAdmin(anotherSn); client = connection.getClient(anotherSn); assertFalse(admin instanceof RSRpcServices); assertFalse(client instanceof RSRpcServices); assertTrue(connection.getAdmin().getConnection() == connection); } }
@Test public void testLocations() throws Exception { byte[] b1 = "testLocations".getBytes(); openRegion(hriSecondary); ClusterConnection hc = (ClusterConnection) HTU.getAdmin().getConnection(); try { hc.clearRegionCache(); RegionLocations rl = hc.locateRegion(table.getName(), b1, false, false); Assert.assertEquals(2, rl.size()); rl = hc.locateRegion(table.getName(), b1, true, false); Assert.assertEquals(2, rl.size()); hc.clearRegionCache(); rl = hc.locateRegion(table.getName(), b1, true, false); Assert.assertEquals(2, rl.size()); rl = hc.locateRegion(table.getName(), b1, false, false); Assert.assertEquals(2, rl.size()); } finally { closeRegion(hriSecondary); } }