public synchronized Optional<ServerName> acquire() { List<ServerName> serverList = master.getServerManager().getOnlineServersList(); Collections.shuffle(serverList); Optional<ServerName> worker = serverList.stream().filter( serverName -> !currentWorkers.containsKey(serverName) || currentWorkers.get(serverName) > 0) .findAny(); if (worker.isPresent()) { currentWorkers.compute(worker.get(), (serverName, availableWorker) -> availableWorker == null ? maxSplitTasks - 1 : availableWorker - 1); } return worker; }
private List<ServerName> findIdleServers(List<ServerName> servers) { return this.services.getServerManager() .getOnlineServersListWithPredicator(servers, IDLE_SERVER_PREDICATOR); }
/** * @param master the master to get online servers for * @return Set of online Servers named for their hostname and port (not ServerName). */ private static Set<Address> getOnlineServers(final MasterServices master) { Set<Address> onlineServers = new HashSet<Address>(); if (master == null) { return onlineServers; } for (ServerName server : master.getServerManager().getOnlineServers().keySet()) { onlineServers.add(server.getAddress()); } return onlineServers; }
protected AdminService.BlockingInterface getRsAdmin() throws IOException { final AdminService.BlockingInterface admin = master.getServerManager().getRsAdmin(serverName); if (admin == null) { throw new IOException("Attempting to send OPEN RPC to server " + getServerName() + " failed because no RPC connection found to this server"); } return admin; }
private void checkOnlineServersOnly(Set<Address> servers) throws ConstraintException { // This uglyness is because we only have Address, not ServerName. // Online servers are keyed by ServerName. Set<Address> onlineServers = new HashSet<>(); for(ServerName server: master.getServerManager().getOnlineServers().keySet()) { onlineServers.add(server.getAddress()); } for (Address address: servers) { if (!onlineServers.contains(address)) { throw new ConstraintException( "Server " + address + " is not an online server in 'default' RSGroup."); } } }
public SplitWorkerAssigner(MasterServices master, int maxSplitTasks) { this.maxSplitTasks = maxSplitTasks; this.master = master; this.event = new ProcedureEvent<>("split-WAL-worker-assigning"); this.master.getServerManager().registerListener(this); }
@Override public boolean stop() { if (!super.stop()) { return false; } master.getServerManager().unregisterListener(this); return true; }
protected final void refreshPeer(MasterProcedureEnv env, PeerOperationType type) { addChildProcedure(env.getMasterServices().getServerManager().getOnlineServersList().stream() .map(sn -> new RefreshPeerProcedure(peerId, type, sn)).toArray(RefreshPeerProcedure[]::new)); }
public void joinCluster() throws IOException { long startTime = System.nanoTime(); LOG.debug("Joining cluster..."); // Scan hbase:meta to build list of existing regions, servers, and assignment. // hbase:meta is online now or will be. Inside loadMeta, we keep trying. Can't make progress // w/o meta. loadMeta(); while (master.getServerManager().countOfRegionServers() < 1) { LOG.info("Waiting for RegionServers to join; current count={}", master.getServerManager().countOfRegionServers()); Threads.sleep(250); } LOG.info("Number of RegionServers={}", master.getServerManager().countOfRegionServers()); // Start the RIT chore master.getMasterProcedureExecutor().addChore(this.ritChore); long costMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startTime); LOG.info("Joined the cluster in {}", StringUtils.humanTimeDiff(costMs)); }
private List<ServerName> getOnlineRS() throws IOException { if (masterServices != null) { return masterServices.getServerManager().getOnlineServersList(); } LOG.debug("Reading online RS from zookeeper"); List<ServerName> servers = new LinkedList<>(); try { for (String el : ZKUtil.listChildrenNoWatch(watcher, watcher.getZNodePaths().rsZNode)) { servers.add(ServerName.parseServerName(el)); } } catch (KeeperException e) { throw new IOException("Failed to retrieve server list from zookeeper", e); } return servers; }
@Override public boolean start() { if (!super.start()) { return false; } master.getServerManager().registerListener(this); procedureEnv = master.getMasterProcedureExecutor().getEnvironment(); for (ServerName serverName: master.getServerManager().getOnlineServersList()) { addNode(serverName); } return true; }
@Override protected void remoteDispatch(final ServerName serverName, final Set<RemoteProcedure> remoteProcedures) { final int rsVersion = master.getServerManager().getVersionNumber(serverName); if (rsVersion >= RS_VERSION_WITH_EXEC_PROCS) { LOG.trace("Using procedure batch rpc execution for serverName={} version={}", serverName, rsVersion); submitTask(new ExecuteProceduresRemoteCall(serverName, remoteProcedures)); } else if (rsVersion == 0 && !master.getServerManager().isServerOnline(serverName)) { submitTask(new DeadRSRemoteCall(serverName, remoteProcedures)); } else { LOG.info(String.format( "Fallback to compat rpc execution for serverName=%s version=%s", serverName, rsVersion)); submitTask(new CompatRemoteProcedureResolver(serverName, remoteProcedures)); } }
static void removeNonDefaultReplicas(MasterProcedureEnv env, Stream<RegionInfo> regions, int regionReplication) { // Remove from in-memory states regions.flatMap(hri -> IntStream.range(1, regionReplication) .mapToObj(i -> RegionReplicaUtil.getRegionInfoForReplica(hri, i))).forEach(hri -> { env.getAssignmentManager().getRegionStates().deleteRegion(hri); env.getMasterServices().getServerManager().removeRegion(hri); FavoredNodesManager fnm = env.getMasterServices().getFavoredNodesManager(); if (fnm != null) { fnm.deleteFavoredNodesForRegions(Collections.singletonList(hri)); } }); }
private synchronized void init() throws IOException { refresh(); serverEventsListenerThread.start(); masterServices.getServerManager().registerListener(serverEventsListenerThread); failedOpenUpdaterThread = new FailedOpenUpdaterThread(masterServices.getConfiguration()); failedOpenUpdaterThread.start(); masterServices.getServerManager().registerListener(failedOpenUpdaterThread); }
private void updateRegionMergeTransition(final ServerName serverName, final TransitionCode state, final RegionInfo merged, final RegionInfo hriA, final RegionInfo hriB) throws IOException { checkMetaLoaded(merged); if (state != TransitionCode.READY_TO_MERGE) { throw new UnexpectedStateException("Unsupported merge regionState=" + state + " for regionA=" + hriA + " regionB=" + hriB + " merged=" + merged + " maybe an old RS (< 2.0) had the operation in progress"); } // Submit the Merge procedure if (LOG.isDebugEnabled()) { LOG.debug("Handling merge request from RS=" + merged + ", merged=" + merged); } master.getMasterProcedureExecutor().submitProcedure(createMergeProcedure(hriA, hriB)); // If the RS is < 2.0 throw an exception to abort the operation, we are handling the merge if (master.getServerManager().getVersionNumber(serverName) < 0x0200000) { throw new UnsupportedOperationException(String.format( "Merge not handled yet: regionState=%s merged=%s hriA=%s hriB=%s", state, merged, hriA, hriB)); } }
@Override protected void chore() { try { AssignmentManager am = this.services.getAssignmentManager(); if (this.enabled.get() && !this.services.isInMaintenanceMode() && !this.services.getServerManager().isClusterShutdown() && am != null && am.isMetaLoaded() && !am.hasRegionsInTransition()) { scan(); } else { LOG.warn("CatalogJanitor is disabled! Enabled=" + this.enabled.get() + ", maintenanceMode=" + this.services.isInMaintenanceMode() + ", am=" + am + ", metaLoaded=" + (am != null && am.isMetaLoaded()) + ", hasRIT=" + (am != null && am.hasRegionsInTransition()) + " clusterShutDown=" + this.services .getServerManager().isClusterShutdown()); } } catch (IOException e) { LOG.warn("Failed scan of catalog table", e); } }
public SyncReplicationReplayWALManager(MasterServices services) throws IOException, ReplicationException { this.serverManager = services.getServerManager(); this.fs = services.getMasterFileSystem().getWALFileSystem(); this.walRootDir = services.getMasterFileSystem().getWALRootDir(); this.remoteWALDir = new Path(this.walRootDir, ReplicationUtils.REMOTE_WAL_DIR_NAME); serverManager.registerListener(new ServerListener() { @Override public void serverAdded(ServerName serverName) { MasterProcedureScheduler scheduler = services.getMasterProcedureExecutor().getEnvironment().getProcedureScheduler(); for (UsedReplayWorkersForPeer usedWorkers : usedWorkersByPeer.values()) { synchronized (usedWorkers) { usedWorkers.wake(scheduler); } } } }); }
private long getRegionSize(RegionInfo hri) { ServerName sn = masterServices.getAssignmentManager().getRegionStates(). getRegionServerOfRegion(hri); RegionMetrics regionLoad = masterServices.getServerManager().getLoad(sn). getRegionMetrics().get(hri.getRegionName()); if (regionLoad == null) { LOG.debug(hri.getRegionNameAsString() + " was not found in RegionsLoad"); return -1; } return (long) regionLoad.getStoreFileSize().get(Size.Unit.MEGABYTE); } }
protected static void deleteFromMeta(final MasterProcedureEnv env, final TableName tableName, List<RegionInfo> regions) throws IOException { MetaTableAccessor.deleteRegions(env.getMasterServices().getConnection(), regions); // Clean any remaining rows for this table. cleanAnyRemainingRows(env, tableName); // clean region references from the server manager env.getMasterServices().getServerManager().removeRegions(regions); // Clear Favored Nodes for this table FavoredNodesManager fnm = env.getMasterServices().getFavoredNodesManager(); if (fnm != null) { fnm.deleteFavoredNodesForRegions(regions); } }
@Override public Void call() throws Exception { AssignmentManager am = env.getAssignmentManager(); // try to simulate a master restart by removing the ServerManager states about seqIDs for (RegionState regionState: am.getRegionStates().getRegionStates()) { env.getMasterServices().getServerManager().removeRegion(regionState.getRegion()); } am.stop(); master.setInitialized(false); return null; } },