@Override public synchronized void addRSGroup(RSGroupInfo rsGroupInfo) throws IOException { checkGroupName(rsGroupInfo.getName()); if (rsGroupMap.get(rsGroupInfo.getName()) != null || rsGroupInfo.getName().equals(RSGroupInfo.DEFAULT_GROUP)) { throw new DoNotRetryIOException("Group already exists: " + rsGroupInfo.getName()); } Map<String, RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap); newGroupMap.put(rsGroupInfo.getName(), rsGroupInfo); flushConfig(newGroupMap); }
private synchronized void updateDefaultServers(SortedSet<Address> servers) throws IOException { RSGroupInfo info = rsGroupMap.get(RSGroupInfo.DEFAULT_GROUP); RSGroupInfo newInfo = new RSGroupInfo(info.getName(), servers, info.getTables()); HashMap<String, RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap); newGroupMap.put(newInfo.getName(), newInfo); flushConfig(newGroupMap); }
@Override public void postDeleteTable(ObserverContext<MasterCoprocessorEnvironment> ctx, TableName tableName) throws IOException { try { RSGroupInfo group = groupAdminServer.getRSGroupInfoOfTable(tableName); if (group != null) { LOG.debug(String.format("Removing deleted table '%s' from rsgroup '%s'", tableName, group.getName())); groupAdminServer.moveTables(Sets.newHashSet(tableName), null); } } catch (IOException ex) { LOG.debug("Failed to perform RSGroup information cleanup for table: " + tableName, ex); } }
@Override public synchronized void removeServers(Set<Address> servers) throws IOException { Map<String, RSGroupInfo> rsGroupInfos = new HashMap<String, RSGroupInfo>(); for (Address el : servers) { RSGroupInfo rsGroupInfo = getRSGroupOfServer(el); if (rsGroupInfo != null) { RSGroupInfo newRsGroupInfo = rsGroupInfos.get(rsGroupInfo.getName()); if (newRsGroupInfo == null) { rsGroupInfo.removeServer(el); rsGroupInfos.put(rsGroupInfo.getName(), rsGroupInfo); } else { newRsGroupInfo.removeServer(el); rsGroupInfos.put(newRsGroupInfo.getName(), newRsGroupInfo); } } else { LOG.warn("Server " + el + " does not belong to any rsgroup."); } } if (rsGroupInfos.size() > 0) { Map<String, RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap); newGroupMap.putAll(rsGroupInfos); flushConfig(newGroupMap); } }
@Override public synchronized void moveTables(Set<TableName> tableNames, String groupName) throws IOException { if (groupName != null && !rsGroupMap.containsKey(groupName)) { throw new DoNotRetryIOException("Group " + groupName + " does not exist"); } Map<String, RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap); for (TableName tableName : tableNames) { if (tableMap.containsKey(tableName)) { RSGroupInfo src = new RSGroupInfo(newGroupMap.get(tableMap.get(tableName))); src.removeTable(tableName); newGroupMap.put(src.getName(), src); } if (groupName != null) { RSGroupInfo dst = new RSGroupInfo(newGroupMap.get(groupName)); dst.addTable(tableName); newGroupMap.put(dst.getName(), dst); } } flushConfig(newGroupMap); }
private synchronized Map<TableName, String> flushConfigTable(Map<String, RSGroupInfo> groupMap) throws IOException { Map<TableName, String> newTableMap = Maps.newHashMap(); List<Mutation> mutations = Lists.newArrayList(); // populate deletes for (String groupName : prevRSGroups) { if (!groupMap.containsKey(groupName)) { Delete d = new Delete(Bytes.toBytes(groupName)); mutations.add(d); } } // populate puts for (RSGroupInfo RSGroupInfo : groupMap.values()) { RSGroupProtos.RSGroupInfo proto = RSGroupProtobufUtil.toProtoGroupInfo(RSGroupInfo); Put p = new Put(Bytes.toBytes(RSGroupInfo.getName())); p.addColumn(META_FAMILY_BYTES, META_QUALIFIER_BYTES, proto.toByteArray()); mutations.add(p); for (TableName entry : RSGroupInfo.getTables()) { newTableMap.put(entry, RSGroupInfo.getName()); } } if (mutations.size() > 0) { multiMutate(mutations); } return newTableMap; }
@Override public synchronized Set<Address> moveServers(Set<Address> servers, String srcGroup, String dstGroup) throws IOException { RSGroupInfo src = getRSGroupInfo(srcGroup); RSGroupInfo dst = getRSGroupInfo(dstGroup); // If destination is 'default' rsgroup, only add servers that are online. If not online, drop // it. If not 'default' group, add server to 'dst' rsgroup EVEN IF IT IS NOT online (could be a // rsgroup of dead servers that are to come back later). Set<Address> onlineServers = dst.getName().equals(RSGroupInfo.DEFAULT_GROUP) ? getOnlineServers(this.masterServices) : null; for (Address el : servers) { src.removeServer(el); if (onlineServers != null) { if (!onlineServers.contains(el)) { if (LOG.isDebugEnabled()) { LOG.debug("Dropping " + el + " during move-to-default rsgroup because not online"); } continue; } } dst.addServer(el); } Map<String, RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap); newGroupMap.put(src.getName(), src); newGroupMap.put(dst.getName(), dst); flushConfig(newGroupMap); return dst.getServers(); }
" on server: " + assignedServer + " found in group: " + otherInfo + " outside of group: " + (info == null ? "UNKNOWN" : info.getName())); misplacedRegions.add(regionInfo);
@Override public void moveServersAndTables(Set<Address> servers, Set<TableName> tables, String srcGroup, String dstGroup) throws IOException { // get server's group RSGroupInfo srcGroupInfo = getRSGroupInfo(srcGroup); RSGroupInfo dstGroupInfo = getRSGroupInfo(dstGroup); // move servers for (Address el : servers) { srcGroupInfo.removeServer(el); dstGroupInfo.addServer(el); } // move tables for (TableName tableName : tables) { srcGroupInfo.removeTable(tableName); dstGroupInfo.addTable(tableName); } // flush changed groupinfo Map<String, RSGroupInfo> newGroupMap = Maps.newHashMap(rsGroupMap); newGroupMap.put(srcGroupInfo.getName(), srcGroupInfo); newGroupMap.put(dstGroupInfo.getName(), dstGroupInfo); flushConfig(newGroupMap); }
@Override public void moveServersAndTables(Set<Address> servers, Set<TableName> tables, String targetGroup) throws IOException { if (servers == null || servers.isEmpty()) { throw new ConstraintException("The list of servers to move cannot be null or empty."); } if (tables == null || tables.isEmpty()) { throw new ConstraintException("The list of tables to move cannot be null or empty."); } //check target group getAndCheckRSGroupInfo(targetGroup); // Hold a lock on the manager instance while moving servers and tables to prevent // another writer changing our state while we are working. synchronized (rsGroupInfoManager) { //check servers and tables status checkServersAndTables(servers, tables, targetGroup); //Move servers and tables to a new group. String srcGroup = getRSGroupOfServer(servers.iterator().next()).getName(); rsGroupInfoManager.moveServersAndTables(servers, tables, srcGroup, targetGroup); //move regions which should not belong to these tables moveRegionsFromServers(servers, tables, targetGroup); //move regions which should belong to these servers moveRegionsToServers(servers, tables, targetGroup); } LOG.info("Move servers and tables done. Severs :" + servers + " , Tables : " + tables + " => " + targetGroup); }
private SortedSet<Address> getDefaultServers() throws IOException { SortedSet<Address> defaultServers = Sets.newTreeSet(); for (ServerName serverName : getOnlineRS()) { Address server = Address.fromParts(serverName.getHostname(), serverName.getPort()); boolean found = false; for (RSGroupInfo rsgi : listRSGroups()) { if (!RSGroupInfo.DEFAULT_GROUP.equals(rsgi.getName()) && rsgi.containsServer(server)) { found = true; break; } } if (!found) { defaultServers.add(server); } } return defaultServers; }
static RSGroupProtos.RSGroupInfo toProtoGroupInfo(RSGroupInfo pojo) { List<HBaseProtos.TableName> tables = new ArrayList<>(pojo.getTables().size()); for(TableName arg: pojo.getTables()) { tables.add(ProtobufUtil.toProtoTableName(arg)); } List<HBaseProtos.ServerName> hostports = new ArrayList<>(pojo.getServers().size()); for(Address el: pojo.getServers()) { hostports.add(HBaseProtos.ServerName.newBuilder() .setHostName(el.getHostname()) .setPort(el.getPort()) .build()); } return RSGroupProtos.RSGroupInfo.newBuilder().setName(pojo.getName()) .addAllServers(hostports) .addAllTables(tables).build(); } }
@Test public void testDisabledTableMove() throws Exception { final byte[] familyNameBytes = Bytes.toBytes("f"); String newGroupName = getGroupName(name.getMethodName()); final RSGroupInfo newGroup = addGroup(newGroupName, 2); TEST_UTIL.createMultiRegionTable(tableName, familyNameBytes, 5); TEST_UTIL.waitFor(WAIT_TIMEOUT, new Waiter.Predicate<Exception>() { @Override public boolean evaluate() throws Exception { List<String> regions = getTableRegionMap().get(tableName); if (regions == null) { return false; } return getTableRegionMap().get(tableName).size() >= 5; } }); RSGroupInfo tableGrp = rsGroupAdmin.getRSGroupInfoOfTable(tableName); assertTrue(tableGrp.getName().equals(RSGroupInfo.DEFAULT_GROUP)); // test disable table admin.disableTable(tableName); // change table's group LOG.info("Moving table " + tableName + " to " + newGroup.getName()); rsGroupAdmin.moveTables(Sets.newHashSet(tableName), newGroup.getName()); // verify group change Assert.assertEquals(newGroup.getName(), rsGroupAdmin.getRSGroupInfoOfTable(tableName).getName()); }
public void deleteGroups() throws IOException { RSGroupAdmin groupAdmin = new RSGroupAdminClient(TEST_UTIL.getConnection()); for(RSGroupInfo group: groupAdmin.listRSGroups()) { if(!group.getName().equals(RSGroupInfo.DEFAULT_GROUP)) { groupAdmin.moveTables(group.getTables(), RSGroupInfo.DEFAULT_GROUP); groupAdmin.moveServers(group.getServers(), RSGroupInfo.DEFAULT_GROUP); groupAdmin.removeRSGroup(group.getName()); } } }
protected TableName getTableName(ServerName sn) throws IOException { TableName tableName = null; RSGroupInfoManager gm = getMockedGroupInfoManager(); RSGroupInfo groupOfServer = null; for(RSGroupInfo gInfo : gm.listRSGroups()){ if(gInfo.containsServer(sn.getAddress())){ groupOfServer = gInfo; break; } } for(TableDescriptor desc : tableDescs){ if(gm.getRSGroupOfTable(desc.getTableName()).endsWith(groupOfServer.getName())){ tableName = desc.getTableName(); } } return tableName; } }
@Test public void testRegionServerMove() throws IOException, InterruptedException { int initNumGroups = rsGroupAdmin.listRSGroups().size(); RSGroupInfo appInfo = addGroup(getGroupName(name.getMethodName()), 1); RSGroupInfo adminInfo = addGroup(getGroupName(name.getMethodName()), 1); RSGroupInfo dInfo = rsGroupAdmin.getRSGroupInfo(RSGroupInfo.DEFAULT_GROUP); Assert.assertEquals(initNumGroups + 2, rsGroupAdmin.listRSGroups().size()); assertEquals(1, adminInfo.getServers().size()); assertEquals(1, appInfo.getServers().size()); assertEquals(getNumServers() - 2, dInfo.getServers().size()); rsGroupAdmin.moveServers(appInfo.getServers(), RSGroupInfo.DEFAULT_GROUP); rsGroupAdmin.removeRSGroup(appInfo.getName()); rsGroupAdmin.moveServers(adminInfo.getServers(), RSGroupInfo.DEFAULT_GROUP); rsGroupAdmin.removeRSGroup(adminInfo.getName()); Assert.assertEquals(rsGroupAdmin.listRSGroups().size(), initNumGroups); }
protected ArrayListMultimap<String, ServerAndLoad> convertToGroupBasedMap( final Map<ServerName, List<RegionInfo>> serversMap) throws IOException { ArrayListMultimap<String, ServerAndLoad> loadMap = ArrayListMultimap .create(); for (RSGroupInfo gInfo : getMockedGroupInfoManager().listRSGroups()) { Set<Address> groupServers = gInfo.getServers(); for (Address hostPort : groupServers) { ServerName actual = null; for(ServerName entry: servers) { if(entry.getAddress().equals(hostPort)) { actual = entry; break; } } List<RegionInfo> regions = serversMap.get(actual); assertTrue("No load for " + actual, regions != null); loadMap.put(gInfo.getName(), new ServerAndLoad(actual, regions.size())); } } return loadMap; }
@Test public void testMisplacedRegions() throws Exception { final TableName tableName = TableName.valueOf(tablePrefix + "_testMisplacedRegions"); LOG.info("testMisplacedRegions"); final RSGroupInfo RSGroupInfo = addGroup("testMisplacedRegions", 1); TEST_UTIL.createMultiRegionTable(tableName, new byte[] { 'f' }, 15); TEST_UTIL.waitUntilAllRegionsAssigned(tableName); rsGroupAdminEndpoint.getGroupInfoManager().moveTables(Sets.newHashSet(tableName), RSGroupInfo.getName()); admin.balancerSwitch(true, true); assertTrue(rsGroupAdmin.balanceRSGroup(RSGroupInfo.getName())); admin.balancerSwitch(false, true); assertTrue(observer.preBalanceRSGroupCalled); assertTrue(observer.postBalanceRSGroupCalled); TEST_UTIL.waitFor(60000, new Predicate<Exception>() { @Override public boolean evaluate() throws Exception { ServerName serverName = ServerName.valueOf(RSGroupInfo.getServers().iterator().next().toString(), 1); return admin.getConnection().getAdmin().getRegions(serverName).size() == 15; } }); }
@Test public void testFailRemoveGroup() throws IOException, InterruptedException { int initNumGroups = rsGroupAdmin.listRSGroups().size(); addGroup("bar", 3); TEST_UTIL.createTable(tableName, Bytes.toBytes("f")); rsGroupAdmin.moveTables(Sets.newHashSet(tableName), "bar"); RSGroupInfo barGroup = rsGroupAdmin.getRSGroupInfo("bar"); // group is not empty therefore it should fail try { rsGroupAdmin.removeRSGroup(barGroup.getName()); fail("Expected remove group to fail"); } catch (IOException e) { } // group cannot lose all it's servers therefore it should fail try { rsGroupAdmin.moveServers(barGroup.getServers(), RSGroupInfo.DEFAULT_GROUP); fail("Expected move servers to fail"); } catch (IOException e) { } rsGroupAdmin.moveTables(barGroup.getTables(), RSGroupInfo.DEFAULT_GROUP); try { rsGroupAdmin.removeRSGroup(barGroup.getName()); fail("Expected move servers to fail"); } catch (IOException e) { } rsGroupAdmin.moveServers(barGroup.getServers(), RSGroupInfo.DEFAULT_GROUP); rsGroupAdmin.removeRSGroup(barGroup.getName()); Assert.assertEquals(initNumGroups, rsGroupAdmin.listRSGroups().size()); }
assertEquals(0, notClearedServers.size()); Set<Address> newGroupServers = rsGroupAdmin.getRSGroupInfo(newGroup.getName()).getServers(); assertFalse(newGroupServers.contains(targetServer.getAddress())); assertEquals(2, newGroupServers.size());