/** * @see HRegionServer#abort(String, Throwable) */ public void abort(String reason) { abort(reason, null); }
@Override public void uncaughtException(Thread t, Throwable e) { abort("Uncaught exception in executorService thread " + t.getName(), e); } };
@VisibleForTesting protected void kill() { this.killed = true; abort("Simulated kill"); }
private void abortRegionServer(String reason, Throwable cause) { super.abort(reason, cause); } }
/** * Checks to see if the file system is still accessible. If not, sets * abortRequested and stopRequested * * @return false if file system is not available */ public boolean checkFileSystem() { if (this.fsOk && this.fs != null) { try { FSUtils.checkFileSystemAvailable(this.fs); } catch (IOException e) { abort("File System not available", e); this.fsOk = false; } } return this.fsOk; }
@Override public void abort(String reason, Throwable cause) { if (cause instanceof SessionExpiredException) { // called from ZKWatcher, let's wait a bit to make sure that we call stop before calling // abort. try { latch.await(); } catch (InterruptedException e) { } } else { // abort from other classes, usually LogRoller, now we can make progress on abort. latch.countDown(); } super.abort(reason, cause); } }
/** * All initialization needed before we go register with Master.<br> * Do bare minimum. Do bulk of initializations AFTER we've connected to the Master.<br> * In here we just put up the RpcServer, setup Connection, and ZooKeeper. */ private void preRegistrationInitialization() { try { initializeZooKeeper(); setupClusterConnection(); // Setup RPC client for master communication this.rpcClient = RpcClientFactory.createClient(conf, clusterId, new InetSocketAddress( this.rpcServices.isa.getAddress(), 0), clusterConnection.getConnectionMetrics()); } catch (Throwable t) { // Call stop if error or process will stick around for ever since server // puts up non-daemon threads. this.rpcServices.stop(); abort("Initialization of RS failed. Hence aborting RS.", t); } }
/** * Cause a region server to exit doing basic clean up only on its way out. * @param serverNumber Used as index into a list. */ public String abortRegionServer(int serverNumber) { HRegionServer server = getRegionServer(serverNumber); LOG.info("Aborting " + server.toString()); server.abort("Aborting for tests", new Exception("Trace info")); return server.toString(); }
@Test public void testWithRegionServerFailover() throws Exception { AsyncTable<?> table = tableGetter.apply(TABLE_NAME); table.putAll(IntStream.range(0, COUNT) .mapToObj(i -> new Put(getRow(i)).addColumn(FAMILY, CQ, Bytes.toBytes(i))) .collect(Collectors.toList())).get(); TEST_UTIL.getMiniHBaseCluster().getRegionServer(0).abort("Aborting for tests"); Thread.sleep(100); table.putAll(IntStream.range(COUNT, 2 * COUNT) .mapToObj(i -> new Put(getRow(i)).addColumn(FAMILY, CQ, Bytes.toBytes(i))) .collect(Collectors.toList())).get(); List<Result> results = table.getAll( IntStream.range(0, 2 * COUNT).mapToObj(i -> new Get(getRow(i))).collect(Collectors.toList())) .get(); assertEquals(2 * COUNT, results.size()); results.forEach(r -> assertFalse(r.isEmpty())); table.deleteAll(IntStream.range(0, 2 * COUNT).mapToObj(i -> new Delete(getRow(i))) .collect(Collectors.toList())).get(); results = table.getAll( IntStream.range(0, 2 * COUNT).mapToObj(i -> new Get(getRow(i))).collect(Collectors.toList())) .get(); assertEquals(2 * COUNT, results.size()); results.forEach(r -> assertTrue(r.isEmpty())); }
@Override public void run() { List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster() .getRegionServerThreads(); for (RegionServerThread rsThread : regionServerThreads) { List<HRegion> onlineRegions = rsThread.getRegionServer().getRegions( LABELS_TABLE_NAME); if (onlineRegions.size() > 0) { rsThread.getRegionServer().abort("Aborting "); killedRS = true; break; } } }
@Test public void testAbortTimeout() throws Exception { Thread writer = new Thread(() -> { try { try (Table table = UTIL.getConnection().getTable(TABLE_NAME)) { for (int i = 0; i < 10000; i++) { table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); } } } catch (IOException e) { LOG.warn("Failed to load data"); } }); writer.setDaemon(true); writer.start(); // Abort one region server UTIL.getMiniHBaseCluster().getRegionServer(0).abort("Abort RS for test"); long startTime = System.currentTimeMillis(); long timeout = REGIONS_NUM * SLEEP_TIME_WHEN_CLOSE_REGION * 10; while (System.currentTimeMillis() - startTime < timeout) { if (UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size() == 1) { assertTrue("Abort timer task should be scheduled", abortTimeoutTaskScheduled); return; } Threads.sleep(SLEEP_TIME_WHEN_CLOSE_REGION); } fail("Failed to abort a region server in " + timeout + " ms"); }
@Test public void testShutdownWhileBecomingActive() throws InterruptedException { MiniHBaseCluster cluster = UTIL.getHBaseCluster(); HMaster activeMaster = null; HMaster backupMaster = null; for (MasterThread t : cluster.getMasterThreads()) { if (t.getMaster().isActiveMaster()) { activeMaster = t.getMaster(); } else { backupMaster = t.getMaster(); } } assertNotNull(activeMaster); assertNotNull(backupMaster); ARRIVE = new CountDownLatch(1); CONTINUE = new CountDownLatch(1); activeMaster.abort("Aborting active master for test"); // wait until we arrive the initClusterSchemaService ARRIVE.await(); // killall RSes cluster.getRegionServerThreads().stream().map(t -> t.getRegionServer()) .forEachOrdered(rs -> rs.abort("Aborting RS for test")); CONTINUE.countDown(); } }
@Override public void run() { try { boolean aborted = false; for (RegionServerThread rs : HTU.getMiniHBaseCluster().getRegionServerThreads()) { for (Region r : rs.getRegionServer().getRegions(htd.getTableName())) { if (r.getRegionInfo().getReplicaId() == 1) { LOG.info("Aborting region server hosting secondary region replica"); rs.getRegionServer().abort("for test"); aborted = true; } } } assertTrue(aborted); } catch (Throwable e) { ex.compareAndSet(null, e); } } };
@Test public void testBatchWithPut() throws Exception { LOG.info("test=testBatchWithPut"); Table table = CONNECTION.getTable(TEST_TABLE); // put multiple rows using a batch List<Put> puts = constructPutRequests(); Object[] results = new Object[puts.size()]; table.batch(puts, results); validateSizeAndEmpty(results, KEYS.length); if (true) { int liveRScount = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().size(); assert liveRScount > 0; JVMClusterUtil.RegionServerThread liveRS = UTIL.getMiniHBaseCluster().getLiveRegionServerThreads().get(0); liveRS.getRegionServer().abort("Aborting for tests", new Exception("testBatchWithPut")); puts = constructPutRequests(); try { results = new Object[puts.size()]; table.batch(puts, results); } catch (RetriesExhaustedWithDetailsException ree) { LOG.info(ree.getExhaustiveDescription()); table.close(); throw ree; } validateSizeAndEmpty(results, KEYS.length); } validateLoadedData(table); table.close(); }
@Test public void testKillRS() throws Exception { TableName tableName = TableName.valueOf(name.getMethodName()); UTIL.getAdmin().createTable( TableDescriptorBuilder.newBuilder(tableName).setColumnFamily(ColumnFamilyDescriptorBuilder .newBuilder(CF).setScope(HConstants.REPLICATION_SCOPE_GLOBAL).build()).build()); UTIL.waitTableAvailable(tableName); try (Table table = UTIL.getConnection().getTable(tableName)) { for (int i = 0; i < 100; i++) { table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); } } RegionServerThread thread = UTIL.getMiniHBaseCluster().getRegionServerThreads().stream() .filter(t -> !t.getRegionServer().getRegions(tableName).isEmpty()).findFirst().get(); thread.getRegionServer().abort("for testing"); thread.join(); try (Table table = UTIL.getConnection().getTable(tableName)) { for (int i = 100; i < 200; i++) { table.put(new Put(Bytes.toBytes(i)).addColumn(CF, CQ, Bytes.toBytes(i))); } } enablePeerAndWaitUntilReplicationDone(200); checkOrder(200); } }
@Test public void test() throws InterruptedException, IOException { HMaster master = UTIL.getMiniHBaseCluster().stopMaster(0).getMaster(); // Shutdown master before shutting down rs UTIL.waitFor(30000, () -> !master.isAlive()); RegionServerThread thread = null; for (RegionServerThread t : UTIL.getMiniHBaseCluster().getRegionServerThreads()) { if (!t.getRegionServer().getRegions(TABLE_NAME).isEmpty()) { thread = t; break; } } // shutdown rs thread.getRegionServer().abort("For testing"); thread.join(); // restart master UTIL.getMiniHBaseCluster().startMaster(); // make sure that we can schedule a SCP for the crashed server which WAL is disabled and bring // the region online. try (Table table = UTIL.getConnection().getTableBuilder(TABLE_NAME, null).setOperationTimeout(30000).build()) { table.put(new Put(Bytes.toBytes(1)).addColumn(CF, CQ, Bytes.toBytes(1))); assertEquals(1, Bytes.toInt(table.get(new Get(Bytes.toBytes(1))).getValue(CF, CQ))); } } }
@Test public void testIsMetaWhenMetaGoesOffline() throws InterruptedException { HMaster m = UTIL.getMiniHBaseCluster().getMaster(); int index = UTIL.getMiniHBaseCluster().getServerWithMeta(); HRegionServer rsWithMeta = UTIL.getMiniHBaseCluster().getRegionServer(index); rsWithMeta.abort("TESTING"); assertTrue(m.waitForMetaOnline()); }
@Test public void testVisibilityLabelsOnRSRestart() throws Exception { final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster() .getRegionServerThreads(); for (RegionServerThread rsThread : regionServerThreads) { rsThread.getRegionServer().abort("Aborting "); } // Start one new RS RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer(); waitForLabelsRegionAvailability(rs.getRegionServer()); try (Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL + ")", PRIVATE)) { Scan s = new Scan(); s.setAuthorizations(new Authorizations(SECRET)); ResultScanner scanner = table.getScanner(s); Result[] next = scanner.next(3); assertTrue(next.length == 1); } }
@Test public void testVisibilityLabelsOnWALReplay() throws Exception { final TableName tableName = TableName.valueOf(TEST_NAME.getMethodName()); try (Table table = createTableAndWriteDataWithLabels(tableName, "(" + SECRET + "|" + CONFIDENTIAL + ")", PRIVATE)) { List<RegionServerThread> regionServerThreads = TEST_UTIL.getHBaseCluster() .getRegionServerThreads(); for (RegionServerThread rsThread : regionServerThreads) { rsThread.getRegionServer().abort("Aborting "); } // Start one new RS RegionServerThread rs = TEST_UTIL.getHBaseCluster().startRegionServer(); waitForLabelsRegionAvailability(rs.getRegionServer()); Scan s = new Scan(); s.setAuthorizations(new Authorizations(SECRET)); ResultScanner scanner = table.getScanner(s); Result[] next = scanner.next(3); assertTrue(next.length == 1); } } }
/** * Do a simulated kill all masters and regionservers. Useful when it is * impossible to bring the mini-cluster back for clean shutdown. */ public void killAll() { // Do backups first. MasterThread activeMaster = null; for (MasterThread masterThread : getMasterThreads()) { if (!masterThread.getMaster().isActiveMaster()) { masterThread.getMaster().abort("killAll"); } else { activeMaster = masterThread; } } // Do active after. if (activeMaster != null) { activeMaster.getMaster().abort("killAll"); } for (RegionServerThread rst : getRegionServerThreads()) { rst.getRegionServer().abort("killAll"); } }