/** * Find regionserver other than the one passed. * Can't rely on indexes into list of regionservers since crashed servers * occupy an index. * @param cluster * @param notThisOne * @return A regionserver that is not <code>notThisOne</code> or null if none * found */ private HRegionServer getOtherRegionServer(final MiniHBaseCluster cluster, final HRegionServer notThisOne) { for (RegionServerThread rst: cluster.getRegionServerThreads()) { HRegionServer hrs = rst.getRegionServer(); if (hrs.getServerName().equals(notThisOne.getServerName())) continue; if (hrs.isStopping() || hrs.isStopped()) continue; return hrs; } return null; }
public String[] getRegionServerCoprocessors() { TreeSet<String> coprocessors = new TreeSet<>(); try { coprocessors.addAll(getWAL(null).getCoprocessorHost().getCoprocessors()); } catch (IOException exception) { LOG.warn("Exception attempting to fetch wal coprocessor information for the common wal; " + "skipping."); LOG.debug("Exception details for failure to fetch wal coprocessor information.", exception); } Collection<HRegion> regions = getOnlineRegionsLocalContext(); for (HRegion region: regions) { coprocessors.addAll(region.getCoprocessorHost().getCoprocessors()); try { coprocessors.addAll(getWAL(region.getRegionInfo()).getCoprocessorHost().getCoprocessors()); } catch (IOException exception) { LOG.warn("Exception attempting to fetch wal coprocessor information for region " + region + "; skipping."); LOG.debug("Exception details for failure to fetch wal coprocessor information.", exception); } } coprocessors.addAll(rsHost.getCoprocessors()); return coprocessors.toArray(new String[coprocessors.size()]); }
private ServerName findRegionLocation(TableName tableName, byte[] startKey, int replicaId) { return UTIL.getMiniHBaseCluster().getRegionServerThreads().stream() .map(t -> t.getRegionServer()) .filter(rs -> rs.getRegions(tableName).stream().map(Region::getRegionInfo) .anyMatch(r -> r.containsRow(startKey) && r.getReplicaId() == replicaId)) .findFirst().get().getServerName(); }
@Override public void run() { if (this.server.isStopping() || this.server.isStopped()) { LOG.debug("Skipping split because server is stopping=" + this.server.isStopping() + " or stopped=" + this.server.isStopped()); return; } doSplitting(); } }
/** * All initialization needed before we go register with Master.<br> * Do bare minimum. Do bulk of initializations AFTER we've connected to the Master.<br> * In here we just put up the RpcServer, setup Connection, and ZooKeeper. */ private void preRegistrationInitialization() { try { initializeZooKeeper(); setupClusterConnection(); // Setup RPC client for master communication this.rpcClient = RpcClientFactory.createClient(conf, clusterId, new InetSocketAddress( this.rpcServices.isa.getAddress(), 0), clusterConnection.getConnectionMetrics()); } catch (Throwable t) { // Call stop if error or process will stick around for ever since server // puts up non-daemon threads. this.rpcServices.stop(); abort("Initialization of RS failed. Hence aborting RS.", t); } }
if (server.isStopping() || server.isStopped() || server.isAborted()) { LOG.info("\t== Offline: " + server.getServerName()); continue; List<HRegionInfo> regions = server.getOnlineRegions(); LOG.info("\t" + server.getServerName() + " regions: " + regions);
static public void flushAllOnlineRegions(HRegionServer regionServer, TableName tableName) throws IOException { for (Region r : regionServer.getOnlineRegions(tableName)) { r.flush(true); } }
if (server.isStopping() || server.isStopped() || server.isAborted()) { LOG.info("\t== Offline: " + server.getServerName()); continue; List<HRegion> regions = server.getRegions(); LOG.info("\t" + server.getServerName() + " regions: " + regions);
this.fsOk = true; this.masterless = conf.getBoolean(MASTERLESS_CONFIG_NAME, false); this.eventLoopGroupConfig = setupNetty(this.conf); MemorySizeUtil.checkForClusterFreeHeapMemoryLimit(this.conf); HFile.checkHFileVersion(this.conf); checkCodecs(this.conf); this.userProvider = UserProvider.instantiate(conf); FSUtils.setupShortCircuitRead(this.conf); rpcServices = createRpcServices(); useThisHostnameInstead = getUseThisHostnameInstead(conf); String hostName = StringUtils.isBlank(useThisHostnameInstead) ? this.rpcServices.isa.getHostName() login(userProvider, hostName); initializeFileSystem(); spanReceiverHost = SpanReceiverHost.getInstance(getConfiguration()); setupWindows(getConfiguration(), getConfigurationManager()); zooKeeper = new ZKWatcher(conf, getProcessName() + ":" + rpcServices.isa.getPort(), this, canCreateBaseZNode()); masterAddressTracker = new MasterAddressTracker(getZooKeeper(), this); masterAddressTracker.start(); this.choreService = new ChoreService(getName(), true); this.executorService = new ExecutorService(getName());
@Test public void testRollWALWALWriter() throws Exception { setUpforLogRolling(); String className = this.getClass().getName(); StringBuilder v = new StringBuilder(className); while (v.length() < 1000) { v.append(className); } byte[] value = Bytes.toBytes(v.toString()); HRegionServer regionServer = startAndWriteData(tableName, value); LOG.info("after writing there are " + AbstractFSWALProvider.getNumRolledLogFiles(regionServer.getWAL(null)) + " log files"); // flush all regions for (HRegion r : regionServer.getOnlineRegionsLocalContext()) { r.flush(true); } admin.rollWALWriter(regionServer.getServerName()).join(); int count = AbstractFSWALProvider.getNumRolledLogFiles(regionServer.getWAL(null)); LOG.info("after flushing all regions and rolling logs there are " + count + " log files"); assertTrue(("actual count: " + count), count <= 2); }
/** * @param miniHBaseCluster * @param server * @param bs * @return */ private List<HRegion> getRegionsFromServerForTable(MiniHBaseCluster cluster, ServerName server, byte[] table) { List<HRegion> online = Collections.emptyList(); for (RegionServerThread rst : cluster.getRegionServerThreads()) { // if its the server we are going to kill, get the regions we want to reassign if (rst.getRegionServer().getServerName().equals(server)) { online = rst.getRegionServer().getOnlineRegions(table); break; } } return online; }
__jamon__var_7.renderNoFlush(jamonWriter, new CacheConfig(regionServer.getConfiguration()), regionServer.getConfiguration(), bcn, bcv, regionServer.getBlockCache().orElse(null) ); ServerInfo serverInfo = ProtobufUtil.getServerInfo(null, regionServer.getRSRpcServices()); ServerName serverName = ProtobufUtil.toServerName(serverInfo.getServerName()); List<RegionInfo> onlineRegions = ProtobufUtil.getOnlineRegions(regionServer.getRSRpcServices()); MasterAddressTracker masterAddressTracker = regionServer.getMasterAddressTracker(); ServerName masterServerName = masterAddressTracker == null ? null : masterAddressTracker.getMasterAddress(); __jamon__var_9.renderNoFlush(jamonWriter, regionServer.getRegionServerMetrics().getRegionServerWrapper(), regionServer.getRpcServer().getMetrics().getHBaseServerWrapper()); __jamon__var_10.renderNoFlush(jamonWriter, new CacheConfig(regionServer.getConfiguration()), regionServer.getConfiguration(), regionServer.getBlockCache().orElse(null) ); org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(regionServer.getZooKeeper().getQuorum()), jamonWriter); org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(java.util.Arrays.toString(regionServer.getRegionServerCoprocessors())), jamonWriter); org.jamon.escaping.Escaping.HTML.write(org.jamon.emit.StandardEmitter.valueOf(new Date(regionServer.getStartcode())), jamonWriter);
/** * The server name in group does not contain the start code, this method will find out the start * code and construct the ServerName object. */ protected ServerName getServerName(Address addr) { return TEST_UTIL.getMiniHBaseCluster().getRegionServerThreads().stream() .map(t -> t.getRegionServer().getServerName()).filter(sn -> sn.getAddress().equals(addr)) .findFirst().get(); }
private void verifyMethodResult(Class<?> coprocessor, String methodName[], TableName tableName, Object value[]) throws IOException { try { for (JVMClusterUtil.RegionServerThread t : cluster.getRegionServerThreads()) { if (!t.isAlive() || t.getRegionServer().isAborted() || t.getRegionServer().isStopping()) { continue; } for (RegionInfo r : ProtobufUtil .getOnlineRegions(t.getRegionServer().getRSRpcServices())) { if (!r.getTable().equals(tableName)) { continue; } RegionCoprocessorHost cph = t.getRegionServer().getOnlineRegion(r.getRegionName()).getCoprocessorHost(); Coprocessor cp = cph.findCoprocessor(coprocessor.getName()); assertNotNull(cp); for (int i = 0; i < methodName.length; ++i) { Method m = coprocessor.getMethod(methodName[i]); Object o = m.invoke(cp); assertTrue("Result of " + coprocessor.getName() + "." + methodName[i] + " is expected to be " + value[i].toString() + ", while we get " + o.toString(), o.equals(value[i])); } } } } catch (Exception e) { throw new IOException(e.toString()); } }
@Before public void setupBasicMocks() throws IOException, ServiceException { rs = Mockito.mock(HRegionServer.class); rpcServices = Mockito.mock(RSRpcServices.class); rpcServer = Mockito.mock(RpcServerInterface.class); Mockito.doReturn(HBaseConfiguration.create()) .when(rs).getConfiguration(); Mockito.doReturn(rpcServices).when(rs).getRSRpcServices(); Mockito.doReturn(rpcServer).when(rs).getRpcServer(); Mockito.doReturn(fakeResponse).when(rpcServices).getServerInfo( (RpcController)Mockito.any(), (GetServerInfoRequest)Mockito.any()); // Fake ZKW ZKWatcher zkw = Mockito.mock(ZKWatcher.class); Mockito.doReturn("fakequorum").when(zkw).getQuorum(); Mockito.doReturn(zkw).when(rs).getZooKeeper(); // Fake BlockCache LOG.warn("The " + HConstants.HFILE_BLOCK_CACHE_SIZE_KEY + " is set to 0"); Mockito.doReturn(Optional.empty()).when(rs).getBlockCache(); // Fake MasterAddressTracker MasterAddressTracker mat = Mockito.mock(MasterAddressTracker.class); Mockito.doReturn(fakeMasterAddress).when(mat).getMasterAddress(); Mockito.doReturn(mat).when(rs).getMasterAddressTracker(); MetricsRegionServer rms = Mockito.mock(MetricsRegionServer.class); Mockito.doReturn(new MetricsRegionServerWrapperStub()).when(rms).getRegionServerWrapper(); Mockito.doReturn(rms).when(rs).getRegionServerMetrics(); MetricsHBaseServer ms = Mockito.mock(MetricsHBaseServer.class); Mockito.doReturn(new MetricsHBaseServerWrapperStub()).when(ms).getHBaseServerWrapper(); Mockito.doReturn(ms).when(rpcServer).getMetrics(); }
/** * Test that a coprocessor is able to override a normal regionserver stop request. */ @Test public void testStopOverrideFromCoprocessor() throws Exception { Admin admin = testUtil.getHBaseAdmin(); HRegionServer regionserver = cluster.getRegionServer(0); admin.stopRegionServer(regionserver.getServerName().getHostAndPort()); // regionserver should have failed to stop due to coprocessor assertFalse(cluster.getRegionServer(0).isAborted()); assertFalse(cluster.getRegionServer(0).isStopped()); }
long globalMemStoreSize = 0; for (RegionInfo regionInfo : ProtobufUtil.getOnlineRegions(null, server.getRSRpcServices())) { globalMemStoreSize += server.getRegion(regionInfo.getEncodedName()).getMemStoreDataSize(); assertEquals(server.getRegionServerAccounting().getGlobalMemStoreDataSize(), globalMemStoreSize); LOG.info("Starting flushes on " + server.getServerName() + ", size=" + server.getRegionServerAccounting().getGlobalMemStoreDataSize()); ProtobufUtil.getOnlineRegions(null, server.getRSRpcServices())) { HRegion r = server.getRegion(regionInfo.getEncodedName()); flush(r, server); LOG.info("Post flush on " + server.getServerName()); long now = System.currentTimeMillis(); long timeout = now + 1000; while(server.getRegionServerAccounting().getGlobalMemStoreDataSize() != 0 && timeout < System.currentTimeMillis()) { Threads.sleep(10); long size = server.getRegionServerAccounting().getGlobalMemStoreDataSize(); if (size > 0) { ProtobufUtil.getOnlineRegions(null, server.getRSRpcServices())) { HRegion r = server.getRegion(regionInfo.getEncodedName()); long l = r.getMemStoreDataSize(); if (l > 0) {
/** * Tests that logs are deleted */ @Test public void testLogRolling() throws Exception { this.tableName = getName(); // TODO: Why does this write data take for ever? startAndWriteData(); RegionInfo region = server.getRegions(TableName.valueOf(tableName)).get(0).getRegionInfo(); final WAL log = server.getWAL(region); LOG.info("after writing there are " + AbstractFSWALProvider.getNumRolledLogFiles(log) + " log files"); assertLogFileSize(log); // flush all regions for (HRegion r : server.getOnlineRegionsLocalContext()) { r.flush(true); } // Now roll the log log.rollWriter(); int count = AbstractFSWALProvider.getNumRolledLogFiles(log); LOG.info("after flushing all regions and rolling logs there are " + count + " log files"); assertTrue(("actual count: " + count), count <= 2); assertLogFileSize(log); }
ZKWatcher zkw = hrs.getZooKeeper(); ServerName sn = MetaTableLocator.getMetaRegionLocation(zkw); if (sn != null && !masterAddr.equals(sn)) { ProtobufUtil.openRegion(null, hrs.getRSRpcServices(), hrs.getServerName(), HRegionInfo.FIRST_META_REGIONINFO); while (true) { sn = MetaTableLocator.getMetaRegionLocation(zkw); if (sn != null && sn.equals(hrs.getServerName()) && hrs.onlineRegions.containsKey( HRegionInfo.FIRST_META_REGIONINFO.getEncodedName())) {