@Override public synchronized byte [] getEncodedNameAsBytes() { if (this.encodedNameAsBytes == null) { this.encodedNameAsBytes = Bytes.toBytes(getEncodedName()); } return this.encodedNameAsBytes; }
/** * @return Return a short, printable name for this region (usually encoded name) for us logging. */ @Override public String getShortNameToLog() { return prettyPrint(this.getEncodedName()); }
/** * Delete the region directory if exists. * @param conf * @param hri * @return True if deleted the region directory. * @throws IOException */ public static boolean deleteRegionDir(final Configuration conf, final HRegionInfo hri) throws IOException { Path rootDir = getRootDir(conf); FileSystem fs = rootDir.getFileSystem(conf); return deleteDirectory(fs, new Path(getTableDir(rootDir, hri.getTable()), hri.getEncodedName())); }
/** * @return Region name as a String for use in logging, etc. */ @Override public String getRegionNameAsString() { if (RegionInfo.hasEncodedName(this.regionName)) { // new format region names already have their encoded name. return Bytes.toStringBinary(this.regionName); } // old format. regionNameStr doesn't have the region name. // // return Bytes.toStringBinary(this.regionName) + "." + this.getEncodedName(); }
@Override public boolean visit(Result data) throws IOException { RegionInfo info = MetaTableAccessor.getRegionInfo(data); if (info == null) { LOG.warn("No serialized HRegionInfo in " + data); return true; } RegionLocations rl = MetaTableAccessor.getRegionLocations(data); boolean matched = false; ServerName sn = null; if (rl != null) { for (HRegionLocation h : rl.getRegionLocations()) { if (h != null && encodedName.equals(h.getRegionInfo().getEncodedName())) { sn = h.getServerName(); info = h.getRegionInfo(); matched = true; } } } if (!matched) return true; result.set(new Pair<>(info, sn)); return false; // found the region, stop } };
/** * @see java.lang.Object#toString() */ @Override public String toString() { return "{ENCODED => " + getEncodedName() + ", " + HConstants.NAME + " => '" + Bytes.toStringBinary(this.regionName) + "', STARTKEY => '" + Bytes.toStringBinary(this.startKey) + "', ENDKEY => '" + Bytes.toStringBinary(this.endKey) + "'" + (isOffline()? ", OFFLINE => true": "") + (isSplit()? ", SPLIT => true": "") + ((replicaId > 0)? ", REPLICA_ID => " + replicaId : "") + "}"; }
private boolean isRegionOpened(HRegionInfo hri) throws Exception { return getRS().getRegionByEncodedName(hri.getEncodedName()).isAvailable(); }
/** Flush the given region in the mini cluster. Since no master, we cannot use HBaseAdmin.flush() */ public static void flushRegion(HBaseTestingUtility HTU, HRegionInfo regionInfo) throws IOException { for (RegionServerThread rst : HTU.getMiniHBaseCluster().getRegionServerThreads()) { HRegion region = rst.getRegionServer().getRegionByEncodedName(regionInfo.getEncodedName()); if (region != null) { region.flush(true); return; } } throw new IOException("Region to flush cannot be found"); }
/** * Test RegionInfo serialization * @throws Exception */ @Test public void testRegionInfo() throws Exception { HRegionInfo hri = createRandomRegion("testRegionInfo"); //test toByteArray() byte [] hrib = hri.toByteArray(); HRegionInfo deserializedHri = HRegionInfo.parseFrom(hrib); assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName()); assertEquals(hri, deserializedHri); //test toDelimitedByteArray() hrib = hri.toDelimitedByteArray(); DataInputBuffer buf = new DataInputBuffer(); try { buf.reset(hrib, hrib.length); deserializedHri = HRegionInfo.parseFrom(buf); assertEquals(hri.getEncodedName(), deserializedHri.getEncodedName()); assertEquals(hri, deserializedHri); } finally { buf.close(); } }
@Test public void testFindsSnapshotFilesWhenCleaning() throws IOException { Configuration conf = TEST_UTIL.getConfiguration(); FSUtils.setRootDir(conf, TEST_UTIL.getDataTestDir()); Path rootDir = FSUtils.getRootDir(conf); Path archivedHfileDir = new Path(TEST_UTIL.getDataTestDir(), HConstants.HFILE_ARCHIVE_DIRECTORY); FileSystem fs = FileSystem.get(conf); SnapshotHFileCleaner cleaner = new SnapshotHFileCleaner(); cleaner.setConf(conf); // write an hfile to the snapshot directory String snapshotName = "snapshot"; byte[] snapshot = Bytes.toBytes(snapshotName); final TableName tableName = TableName.valueOf(name.getMethodName()); Path snapshotDir = SnapshotDescriptionUtils.getCompletedSnapshotDir(snapshotName, rootDir); HRegionInfo mockRegion = new HRegionInfo(tableName); Path regionSnapshotDir = new Path(snapshotDir, mockRegion.getEncodedName()); Path familyDir = new Path(regionSnapshotDir, "family"); // create a reference to a supposedly valid hfile String hfile = "fd1e73e8a96c486090c5cec07b4894c4"; Path refFile = new Path(familyDir, hfile); // make sure the reference file exists fs.create(refFile); // create the hfile in the archive fs.mkdirs(archivedHfileDir); fs.createNewFile(new Path(archivedHfileDir, hfile)); // make sure that the file isn't deletable assertFalse(cleaner.isFileDeletable(fs.getFileStatus(refFile))); }
private Path createReferences(final MasterServices services, final TableDescriptor td, final HRegionInfo parent, final HRegionInfo daughter, final byte [] midkey, final boolean top) throws IOException { Path rootdir = services.getMasterFileSystem().getRootDir(); Path tabledir = FSUtils.getTableDir(rootdir, parent.getTable()); Path storedir = HStore.getStoreHomedir(tabledir, daughter, td.getColumnFamilies()[0].getName()); Reference ref = top? Reference.createTopReference(midkey): Reference.createBottomReference(midkey); long now = System.currentTimeMillis(); // Reference name has this format: StoreFile#REF_NAME_PARSER Path p = new Path(storedir, Long.toString(now) + "." + parent.getEncodedName()); FileSystem fs = services.getMasterFileSystem().getFileSystem(); ref.write(fs, p); return p; }
.setWalGroupId(fakeWalGroupId)); Region region = rs0.getRegion(hriSecondary.getEncodedName()); HTU.verifyNumericRows(region, f, 0, 1000);
@Test public void testGetOnTargetRegionReplica() throws Exception { try { //load some data to primary HTU.loadNumericRows(table, f, 0, 1000); // assert that we can read back from primary Assert.assertEquals(1000, HTU.countRows(table)); // flush so that region replica can read HRegion region = getRS().getRegionByEncodedName(hriPrimary.getEncodedName()); region.flush(true); openRegion(HTU, getRS(), hriSecondary); // try directly Get against region replica byte[] row = Bytes.toBytes(String.valueOf(42)); Get get = new Get(row); get.setConsistency(Consistency.TIMELINE); get.setReplicaId(1); Result result = table.get(get); Assert.assertArrayEquals(row, result.getValue(f, null)); } finally { HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, 0, 1000); closeRegion(HTU, getRS(), hriSecondary); } }
private Path createRegionManifest() throws IOException { byte[] startKey = Bytes.toBytes("AAAAAA"); byte[] stopKey = Bytes.toBytes("BBBBBB"); HRegionInfo regionInfo = new HRegionInfo(TABLE_NAME, startKey, stopKey, false); SnapshotRegionManifest.Builder dataRegionManifestBuilder = SnapshotRegionManifest.newBuilder(); dataRegionManifestBuilder.setRegionInfo(HRegionInfo.convert(regionInfo)); for (ColumnFamilyDescriptor hcd: builder.getTableDescriptor().getColumnFamilies()) { SnapshotRegionManifest.FamilyFiles.Builder family = SnapshotRegionManifest.FamilyFiles.newBuilder(); family.setFamilyName(UnsafeByteOperations.unsafeWrap(hcd.getName())); for (int j = 0; j < TEST_NUM_REGIONFILES; ++j) { SnapshotRegionManifest.StoreFile.Builder sfManifest = SnapshotRegionManifest.StoreFile.newBuilder(); sfManifest.setName(String.format("%064d", j)); sfManifest.setFileSize(j * 1024); family.addStoreFiles(sfManifest.build()); } dataRegionManifestBuilder.addFamilyFiles(family.build()); } SnapshotRegionManifest manifest = dataRegionManifestBuilder.build(); Path regionPath = new Path(snapshotDir, SnapshotManifestV2.SNAPSHOT_MANIFEST_PREFIX + regionInfo.getEncodedName()); FSDataOutputStream stream = fs.create(regionPath); try { manifest.writeTo(stream); } finally { stream.close(); } return regionPath; }
@Test public void testPreWALRestoreSkip() throws Exception { LOG.info(TestRegionObserverInterface.class.getName() + "." + name.getMethodName()); TableName tableName = TableName.valueOf(SimpleRegionObserver.TABLE_SKIPPED); Table table = util.createTable(tableName, new byte[][] { A, B, C }); try (RegionLocator locator = util.getConnection().getRegionLocator(tableName)) { JVMClusterUtil.RegionServerThread rs1 = cluster.startRegionServer(); ServerName sn2 = rs1.getRegionServer().getServerName(); String regEN = locator.getAllRegionLocations().get(0).getRegionInfo().getEncodedName(); util.getAdmin().move(regEN.getBytes(), sn2.getServerName().getBytes()); while (!sn2.equals(locator.getAllRegionLocations().get(0).getServerName())) { Thread.sleep(100); } Put put = new Put(ROW); put.addColumn(A, A, A); put.addColumn(B, B, B); put.addColumn(C, C, C); table.put(put); cluster.killRegionServer(rs1.getRegionServer().getServerName()); Threads.sleep(20000); // just to be sure that the kill has fully started. util.waitUntilAllRegionsAssigned(tableName); } verifyMethodResult(SimpleRegionObserver.class, new String[] { "getCtPreWALRestore", "getCtPostWALRestore", }, tableName, new Integer[] { 0, 0 }); util.deleteTable(tableName); table.close(); }
@Test public void testReplayCallable() throws Exception { // tests replaying the edits to a secondary region replica using the Callable directly openRegion(HTU, rs0, hriSecondary); ClusterConnection connection = (ClusterConnection) ConnectionFactory.createConnection(HTU.getConfiguration()); //load some data to primary HTU.loadNumericRows(table, f, 0, 1000); Assert.assertEquals(1000, entries.size()); // replay the edits to the secondary using replay callable replicateUsingCallable(connection, entries); Region region = rs0.getRegion(hriSecondary.getEncodedName()); HTU.verifyNumericRows(region, f, 0, 1000); HTU.deleteNumericRows(table, f, 0, 1000); closeRegion(HTU, rs0, hriSecondary); connection.close(); }
private void closeRegion(HRegionInfo hri) throws Exception { AdminProtos.CloseRegionRequest crr = ProtobufUtil.buildCloseRegionRequest( getRS().getServerName(), hri.getRegionName()); AdminProtos.CloseRegionResponse responseClose = getRS() .getRSRpcServices().closeRegion(null, crr); Assert.assertTrue(responseClose.getClosed()); checkRegionIsClosed(hri.getEncodedName()); }
@Test public void testRegionReplicaGets() throws Exception { try { //load some data to primary HTU.loadNumericRows(table, f, 0, 1000); // assert that we can read back from primary Assert.assertEquals(1000, HTU.countRows(table)); // flush so that region replica can read HRegion region = getRS().getRegionByEncodedName(hriPrimary.getEncodedName()); region.flush(true); openRegion(HTU, getRS(), hriSecondary); // first try directly against region region = getRS().getRegion(hriSecondary.getEncodedName()); assertGet(region, 42, true); assertGetRpc(hriSecondary, 42, true); } finally { HTU.deleteNumericRows(table, HConstants.CATALOG_FAMILY, 0, 1000); closeRegion(HTU, getRS(), hriSecondary); } }
@Test public void testReplayCallableWithRegionMove() throws Exception { // tests replaying the edits to a secondary region replica using the Callable directly while // the region is moved to another location.It tests handling of RME. openRegion(HTU, rs0, hriSecondary); ClusterConnection connection = (ClusterConnection) ConnectionFactory.createConnection(HTU.getConfiguration()); //load some data to primary HTU.loadNumericRows(table, f, 0, 1000); Assert.assertEquals(1000, entries.size()); // replay the edits to the secondary using replay callable replicateUsingCallable(connection, entries); Region region = rs0.getRegion(hriSecondary.getEncodedName()); HTU.verifyNumericRows(region, f, 0, 1000); HTU.loadNumericRows(table, f, 1000, 2000); // load some more data to primary // move the secondary region from RS0 to RS1 closeRegion(HTU, rs0, hriSecondary); openRegion(HTU, rs1, hriSecondary); // replicate the new data replicateUsingCallable(connection, entries); region = rs1.getRegion(hriSecondary.getEncodedName()); // verify the new data. old data may or may not be there HTU.verifyNumericRows(region, f, 1000, 2000); HTU.deleteNumericRows(table, f, 0, 2000); closeRegion(HTU, rs1, hriSecondary); connection.close(); }
public void assertRegionMetrics(String metric, long expectedValue) throws Exception { try (RegionLocator locator = connection.getRegionLocator(tableName)) { for ( HRegionLocation location: locator.getAllRegionLocations()) { HRegionInfo hri = location.getRegionInfo(); MetricsRegionAggregateSource agg = rs.getRegion(hri.getRegionName()).getMetrics().getSource().getAggregateSource(); String prefix = "namespace_" + NamespaceDescriptor.DEFAULT_NAMESPACE_NAME_STR + "_table_" + tableName.getNameAsString() + "_region_" + hri.getEncodedName()+ "_metric_"; metricsHelper.assertCounter(prefix + metric, expectedValue, agg); } } }