public static RegionLocations getRegionLocations( ClusterConnection connection, TableName tableName, byte[] row, boolean useCache, int replicaId) throws RetriesExhaustedException, DoNotRetryIOException, InterruptedIOException { RegionLocations rl; try { rl = connection.locateRegion(tableName, row, useCache, true, replicaId); } catch (DoNotRetryIOException e) { throw e; } catch (RetriesExhaustedException e) { throw e; } catch (InterruptedIOException e) { throw e; } catch (IOException e) { throw new RetriesExhaustedException("Can't get the location", e); } if (rl == null) { throw new RetriesExhaustedException("Can't get the locations"); } return rl; }
private static void setMockLocation(ClusterConnection hc, byte[] row, RegionLocations result) throws IOException { Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE), Mockito.eq(row), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyInt())).thenReturn(result); Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE), Mockito.eq(row), Mockito.anyBoolean(), Mockito.anyBoolean())).thenReturn(result); }
private RegionLocations findAllLocationsOrFail(Action action, boolean useCache) { if (action.getAction() == null) throw new IllegalArgumentException("#" + asyncProcess.id + ", row cannot be null"); RegionLocations loc = null; try { loc = asyncProcess.connection.locateRegion( tableName, action.getAction().getRow(), useCache, true, action.getReplicaId()); } catch (IOException ex) { manageLocationError(action, ex); } return loc; }
@Test public void testPrepareUsesCache() throws Exception { TableName tableName = TableName.valueOf("MyTable"); Mockito.when(connection.locateRegion(tableName, ROW, true, true, 0)) .thenReturn(regionLocations); ReversedScannerCallable callable = new ReversedScannerCallable(connection, tableName, scan, null, rpcFactory); callable.prepare(false); Mockito.verify(connection).locateRegion(tableName, ROW, true, true, 0); } }
private static void setMockLocation(ClusterConnection hc, byte[] row, RegionLocations result) throws IOException { Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE), Mockito.eq(row), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyInt())).thenReturn(result); Mockito.when(hc.locateRegion(Mockito.eq(DUMMY_TABLE), Mockito.eq(row), Mockito.anyBoolean(), Mockito.anyBoolean())).thenReturn(result); }
static RegionLocations getRegionLocations(boolean useCache, int replicaId, ClusterConnection cConnection, TableName tableName, byte[] row) throws RetriesExhaustedException, DoNotRetryIOException, InterruptedIOException { RegionLocations rl; try { if (useCache) { rl = cConnection.locateRegion(tableName, row, true, true, replicaId); } else { rl = cConnection.relocateRegion(tableName, row, replicaId); } } catch (DoNotRetryIOException | InterruptedIOException | RetriesExhaustedException e) { throw e; } catch (IOException e) { throw new RetriesExhaustedException("Can't get the location for replica " + replicaId, e); } if (rl == null) { throw new RetriesExhaustedException("Can't get the location for replica " + replicaId); } return rl; } }
private void printLocations(Result r) { RegionLocations rl = null; if (r == null) { LOG.info("FAILED FOR null Result"); return; } LOG.info("FAILED FOR " + resultToString(r) + " Stale " + r.isStale()); if (r.getRow() == null) { return; } try { rl = ((ClusterConnection)connection).locateRegion(tableName, r.getRow(), true, true); } catch (IOException e) { LOG.warn("Couldn't get locations for row " + Bytes.toString(r.getRow())); } HRegionLocation locations[] = rl.getRegionLocations(); for (HRegionLocation h : locations) { LOG.info("LOCATION " + h); } }
private ClusterConnection getMockedConnection(final Configuration conf) throws IOException, org.apache.hbase.thirdparty.com.google.protobuf.ServiceException { ClusterConnection c = Mockito.mock(ClusterConnection.class); Mockito.when(c.getConfiguration()).thenReturn(conf); Mockito.doNothing().when(c).close(); // Make it so we return a particular location when asked. final HRegionLocation loc = new HRegionLocation(RegionInfoBuilder.FIRST_META_REGIONINFO, ServerName.valueOf("example.org", 1234, 0)); Mockito.when( c.getRegionLocation((TableName) Mockito.any(), (byte[]) Mockito.any(), Mockito.anyBoolean())) .thenReturn(loc); Mockito.when(c.locateRegion((TableName) Mockito.any(), (byte[]) Mockito.any())).thenReturn(loc); ClientProtos.ClientService.BlockingInterface hri = Mockito.mock(ClientProtos.ClientService.BlockingInterface.class); Mockito .when( hri.bulkLoadHFile((RpcController) Mockito.any(), (BulkLoadHFileRequest) Mockito.any())) .thenThrow(new ServiceException(new IOException("injecting bulk load error"))); Mockito.when(c.getClient(Mockito.any())).thenReturn(hri); return c; }
@Override protected void verifyResultsAndUpdateMetrics(boolean verify, Get[] gets, long elapsedNano, Result[] results, Table table, boolean isNullExpected) throws IOException { super.verifyResultsAndUpdateMetrics(verify, gets, elapsedNano, results, table, isNullExpected); for (Result r : results) { if (r.isStale()) staleReads.incrementAndGet(); } // we actually do not timeout and cancel the reads after timeout. We just wait for the RPC // to complete, but if the request took longer than timeout, we treat that as error. if (elapsedNano > timeoutNano) { timedOutReads.incrementAndGet(); numReadFailures.addAndGet(1); // fail the test for (Result r : results) { LOG.error("FAILED FOR " + r); RegionLocations rl = ((ClusterConnection)connection). locateRegion(tableName, r.getRow(), true, true); HRegionLocation locations[] = rl.getRegionLocations(); for (HRegionLocation h : locations) { LOG.error("LOCATION " + h); } } } } }
/** * Tests the case where all replicas of a region throw an exception. It should not cause a hang * but the exception should propagate to the client */ @Test public void testExceptionsFromReplicasArePropagated() throws IOException { scan.setConsistency(Consistency.TIMELINE); // Mock a caller which calls the callable for ScannerCallableWithReplicas, // but throws an exception for the actual scanner calls via callWithRetries. rpcFactory = new MockRpcRetryingCallerFactory(conf); conf.set(RpcRetryingCallerFactory.CUSTOM_CALLER_CONF_KEY, MockRpcRetryingCallerFactory.class.getName()); // mock 3 replica locations when(clusterConn.locateRegion((TableName)any(), (byte[])any(), anyBoolean(), anyBoolean(), anyInt())).thenReturn(new RegionLocations(null, null, null)); try (MockClientScanner scanner = new MockClientScanner(conf, scan, TableName.valueOf(name.getMethodName()), clusterConn, rpcFactory, new RpcControllerFactory(conf), pool, Integer.MAX_VALUE)) { Iterator<Result> iter = scanner.iterator(); while (iter.hasNext()) { iter.next(); } fail("Should have failed with RetriesExhaustedException"); } catch (RuntimeException expected) { assertThat(expected.getCause(), instanceOf(RetriesExhaustedException.class)); } }
@Test public void testShutdownOfReplicaHolder() throws Exception { // checks that the when the server holding meta replica is shut down, the meta replica // can be recovered try (ClusterConnection conn = (ClusterConnection) ConnectionFactory.createConnection(TEST_UTIL.getConfiguration())) { RegionLocations rl = conn. locateRegion(TableName.META_TABLE_NAME, Bytes.toBytes(""), false, true); HRegionLocation hrl = rl.getRegionLocation(1); ServerName oldServer = hrl.getServerName(); TEST_UTIL.getHBaseClusterInterface().killRegionServer(oldServer); int i = 0; do { LOG.debug("Waiting for the replica " + hrl.getRegionInfo() + " to come up"); Thread.sleep(10000); //wait for the detection/recovery rl = conn.locateRegion(TableName.META_TABLE_NAME, Bytes.toBytes(""), false, true); hrl = rl.getRegionLocation(1); i++; } while ((hrl == null || hrl.getServerName().equals(oldServer)) && i < 3); assertTrue(i != 3); } }
@Nullable private String checkMetaLocationAndExplain(int originalReplicaCount) throws KeeperException, IOException { List<String> metaZnodes = TEST_UTIL.getZooKeeperWatcher().getMetaReplicaNodes(); if (metaZnodes.size() == originalReplicaCount) { RegionLocations rl = ((ClusterConnection) TEST_UTIL.getConnection()) .locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, false, false); for (HRegionLocation location : rl.getRegionLocations()) { if (location == null) { return "Null location found in " + rl.toString(); } if (location.getRegionInfo() == null) { return "Null regionInfo for location " + location; } if (location.getHostname() == null) { return "Null hostName for location " + location; } } return null; // OK } return "Replica count is not as expected " + originalReplicaCount + " <> " + metaZnodes.size() + "(" + metaZnodes.toString() + ")"; }
@Test public void testLocations() throws Exception { byte[] b1 = "testLocations".getBytes(); openRegion(hriSecondary); ClusterConnection hc = (ClusterConnection) HTU.getAdmin().getConnection(); try { hc.clearRegionCache(); RegionLocations rl = hc.locateRegion(table.getName(), b1, false, false); Assert.assertEquals(2, rl.size()); rl = hc.locateRegion(table.getName(), b1, true, false); Assert.assertEquals(2, rl.size()); hc.clearRegionCache(); rl = hc.locateRegion(table.getName(), b1, true, false); Assert.assertEquals(2, rl.size()); rl = hc.locateRegion(table.getName(), b1, false, false); Assert.assertEquals(2, rl.size()); } finally { closeRegion(hriSecondary); } }
RegionLocations rl = connection.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, false, false); if (rl == null) {
private void replicateUsingCallable(ClusterConnection connection, Queue<Entry> entries) throws IOException, RuntimeException { Entry entry; while ((entry = entries.poll()) != null) { byte[] row = CellUtil.cloneRow(entry.getEdit().getCells().get(0)); RegionLocations locations = connection.locateRegion(tableName, row, true, true); RegionReplicaReplayCallable callable = new RegionReplicaReplayCallable(connection, RpcControllerFactory.instantiate(connection.getConfiguration()), table.getName(), locations.getRegionLocation(1), locations.getRegionLocation(1).getRegionInfo(), row, Lists.newArrayList(entry), new AtomicLong()); RpcRetryingCallerFactory factory = RpcRetryingCallerFactory.instantiate( connection.getConfiguration()); factory.<ReplicateWALEntryResponse> newCaller().callWithRetries(callable, 10000); } }
@Test public void testRetryWithExceptionClearsMetaCache() throws Exception { ClusterConnection conn = createHConnection(); Configuration myConf = conn.getConfiguration(); myConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 0); AsyncProcessWithFailure ap = new AsyncProcessWithFailure(conn, myConf, new RegionOpeningException("test")); BufferedMutatorParams bufferParam = createBufferedMutatorParams(ap, DUMMY_TABLE); BufferedMutatorImpl mutator = new BufferedMutatorImpl(conn, bufferParam, ap); Assert.assertNotNull(mutator.getAsyncProcess().createServerErrorTracker()); Assert.assertEquals( conn.locateRegion(DUMMY_TABLE, DUMMY_BYTES_1, true, true).toString(), new RegionLocations(loc1).toString()); Mockito.verify(conn, Mockito.times(0)).clearCaches(Mockito.any()); Put p = createPut(1, true); mutator.mutate(p); try { mutator.flush(); Assert.fail(); } catch (RetriesExhaustedWithDetailsException expected) { assertEquals(1, expected.getNumExceptions()); assertTrue(expected.getRow(0) == p); } Mockito.verify(conn, Mockito.times(1)).clearCaches(loc1.getServerName()); }
.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, false, false); .locateRegion(hdt.getTableName(), row, false, false); .locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, false, false); .locateRegion(hdt.getTableName(), row, false, true);
@Ignore @Test // Disabled. Relies on FSCK which needs work for AMv2. public void testHBaseFsckWithFewerMetaReplicas() throws Exception { ClusterConnection c = (ClusterConnection)ConnectionFactory.createConnection( TEST_UTIL.getConfiguration()); RegionLocations rl = c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, false, false); HBaseFsckRepair.closeRegionSilentlyAndWait(c, rl.getRegionLocation(1).getServerName(), rl.getRegionLocation(1).getRegionInfo()); // check that problem exists HBaseFsck hbck = doFsck(TEST_UTIL.getConfiguration(), false); assertErrors(hbck, new ERROR_CODE[]{ERROR_CODE.UNKNOWN,ERROR_CODE.NO_META_REGION}); // fix the problem hbck = doFsck(TEST_UTIL.getConfiguration(), true); // run hbck again to make sure we don't see any errors hbck = doFsck(TEST_UTIL.getConfiguration(), false); assertErrors(hbck, new ERROR_CODE[]{}); }
@Ignore @Test // The close silently doesn't work any more since HBASE-14614. Fix. public void testHBaseFsckWithFewerMetaReplicaZnodes() throws Exception { ClusterConnection c = (ClusterConnection)ConnectionFactory.createConnection( TEST_UTIL.getConfiguration()); RegionLocations rl = c.locateRegion(TableName.META_TABLE_NAME, HConstants.EMPTY_START_ROW, false, false); HBaseFsckRepair.closeRegionSilentlyAndWait(c, rl.getRegionLocation(2).getServerName(), rl.getRegionLocation(2).getRegionInfo()); ZKWatcher zkw = TEST_UTIL.getZooKeeperWatcher(); ZKUtil.deleteNode(zkw, zkw.getZNodePaths().getZNodeForReplica(2)); // check that problem exists HBaseFsck hbck = doFsck(TEST_UTIL.getConfiguration(), false); assertErrors(hbck, new ERROR_CODE[]{ERROR_CODE.UNKNOWN,ERROR_CODE.NO_META_REGION}); // fix the problem hbck = doFsck(TEST_UTIL.getConfiguration(), true); // run hbck again to make sure we don't see any errors hbck = doFsck(TEST_UTIL.getConfiguration(), false); assertErrors(hbck, new ERROR_CODE[]{}); }
@Test public void testGetRegionLocationFromPrimaryMetaRegion() throws IOException, InterruptedException { HTU.getAdmin().setBalancerRunning(false, true); ((ConnectionImplementation) HTU.getAdmin().getConnection()).setUseMetaReplicas(true); // Create table then get the single region for our new table. HTableDescriptor hdt = HTU.createTableDescriptor("testGetRegionLocationFromPrimaryMetaRegion"); hdt.setRegionReplication(2); try { HTU.createTable(hdt, new byte[][] { f }, null); RegionServerHostingPrimayMetaRegionSlowOrStopCopro.slowDownPrimaryMetaScan = true; // Get user table location, always get it from the primary meta replica RegionLocations url = ((ClusterConnection) HTU.getConnection()) .locateRegion(hdt.getTableName(), row, false, false); } finally { RegionServerHostingPrimayMetaRegionSlowOrStopCopro.slowDownPrimaryMetaScan = false; ((ConnectionImplementation) HTU.getAdmin().getConnection()).setUseMetaReplicas(false); HTU.getAdmin().setBalancerRunning(true, true); HTU.getAdmin().disableTable(hdt.getTableName()); HTU.deleteTable(hdt.getTableName()); } }