.getRegionReplication(); boolean noProblem = true;
@Test public void testMetaHTDReplicaCount() throws Exception { assertTrue(TEST_UTIL.getAdmin().getTableDescriptor(TableName.META_TABLE_NAME) .getRegionReplication() == 3); }
private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException { undeployRegionsForHbi(hi); // undeploy replicas of the region (but only if the method is invoked for the primary) if (hi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) { return; } int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication(); for (int i = 1; i < numReplicas; i++) { if (hi.getPrimaryHRIForDeployedReplica() == null) continue; RegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica( hi.getPrimaryHRIForDeployedReplica(), i); HbckInfo h = regionInfoMap.get(hri.getEncodedName()); if (h != null) { undeployRegionsForHbi(h); //set skip checks; we undeployed it, and we don't want to evaluate this anymore //in consistency checks h.setSkipChecks(true); } } }
int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication(); for (int i = 0; i < numReplicas; i++) { get.addColumn(HConstants.CATALOG_FAMILY, MetaTableAccessor.getServerColumn(i));
|| (!isReadCmd && desc != null && !StringUtils.equals(desc.getRegionSplitPolicyClassName(), opts.splitPolicy)) || (!isReadCmd && desc != null && desc.getRegionReplication() != opts.replicas) || (desc != null && desc.getColumnFamilyCount() != opts.families)) { needsDelete = true;
private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException, KeeperException, InterruptedException { // If we are trying to fix the errors if (shouldFixAssignments()) { errors.print(msg); undeployRegions(hbi); setShouldRerun(); RegionInfo hri = hbi.getHdfsHRI(); if (hri == null) { hri = hbi.metaEntry; } HBaseFsckRepair.fixUnassigned(admin, hri); HBaseFsckRepair.waitUntilAssigned(admin, hri); // also assign replicas if needed (do it only when this call operates on a primary replica) if (hbi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) return; int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication(); for (int i = 1; i < replicationCount; i++) { hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i); HbckInfo h = regionInfoMap.get(hri.getEncodedName()); if (h != null) { undeployRegions(h); //set skip checks; we undeploy & deploy it; we don't want to evaluate this hbi anymore //in consistency checks h.setSkipChecks(true); } HBaseFsckRepair.fixUnassigned(admin, hri); HBaseFsckRepair.waitUntilAssigned(admin, hri); } } }
try { table = master.getConnection().getTable(TableName.valueOf(fqtn)); if (table.getTableDescriptor().getRegionReplication() > 1) { tableHeader = "<h2>Table Regions</h2><table id=\"tableRegionTable\" class=\"tablesorter table table-striped\" style=\"table-layout: fixed; word-wrap: break-word;\"><thead><tr><th>Name</th><th>Region Server</th><th>ReadRequests</th><th>WriteRequests</th><th>StorefileSize</th><th>Num.Storefiles</th><th>MemSize</th><th>Locality</th><th>Start Key</th><th>End Key</th><th>ReplicaID</th></tr></thead>"; withReplica = true;
int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication(); HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(), admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS)) int numReplicas = admin.getTableDescriptor(hbi.getTableName()).getRegionReplication(); HBaseFsckRepair.fixMetaHoleOnlineAndAddReplicas(getConf(), hbi.getHdfsHRI(), admin.getClusterMetrics(EnumSet.of(Option.LIVE_SERVERS))
@Test public void testPb() throws DeserializationException, IOException { HTableDescriptor htd = new HTableDescriptor(TableName.META_TABLE_NAME); final int v = 123; htd.setMaxFileSize(v); htd.setDurability(Durability.ASYNC_WAL); htd.setReadOnly(true); htd.setRegionReplication(2); byte [] bytes = htd.toByteArray(); HTableDescriptor deserializedHtd = HTableDescriptor.parseFrom(bytes); assertEquals(htd, deserializedHtd); assertEquals(v, deserializedHtd.getMaxFileSize()); assertTrue(deserializedHtd.isReadOnly()); assertEquals(Durability.ASYNC_WAL, deserializedHtd.getDurability()); assertEquals(2, deserializedHtd.getRegionReplication()); }
@Test public void testPb() throws DeserializationException, IOException { HTableDescriptor htd = new HTableDescriptor(TableName.META_TABLE_NAME); final int v = 123; htd.setMaxFileSize(v); htd.setDurability(Durability.ASYNC_WAL); htd.setReadOnly(true); htd.setRegionReplication(2); byte [] bytes = htd.toByteArray(); HTableDescriptor deserializedHtd = HTableDescriptor.parseFrom(bytes); assertEquals(htd, deserializedHtd); assertEquals(v, deserializedHtd.getMaxFileSize()); assertTrue(deserializedHtd.isReadOnly()); assertEquals(Durability.ASYNC_WAL, deserializedHtd.getDurability()); assertEquals(2, deserializedHtd.getRegionReplication()); }
/** * Create any replicas for the regions (the default replicas that was * already created is passed to the method) * @param hTableDescriptor * @param regions default replicas * @return the combined list of default and non-default replicas */ protected List<HRegionInfo> addReplicas(HTableDescriptor hTableDescriptor, List<HRegionInfo> regions) { int numRegionReplicas = hTableDescriptor.getRegionReplication() - 1; if (numRegionReplicas <= 0) { return regions; } List<HRegionInfo> hRegionInfos = new ArrayList<HRegionInfo>((numRegionReplicas+1)*regions.size()); for (int i = 0; i < regions.size(); i++) { for (int j = 1; j <= numRegionReplicas; j++) { hRegionInfos.add(RegionReplicaUtil.getRegionInfoForReplica(regions.get(i), j)); } } hRegionInfos.addAll(regions); return hRegionInfos; }
/** * Create any replicas for the regions (the default replicas that was * already created is passed to the method) * @param hTableDescriptor descriptor to use * @param regions default replicas * @return the combined list of default and non-default replicas */ private static List<HRegionInfo> addReplicas(final MasterProcedureEnv env, final HTableDescriptor hTableDescriptor, final List<HRegionInfo> regions) { int numRegionReplicas = hTableDescriptor.getRegionReplication() - 1; if (numRegionReplicas <= 0) { return regions; } List<HRegionInfo> hRegionInfos = new ArrayList<HRegionInfo>((numRegionReplicas+1)*regions.size()); for (int i = 0; i < regions.size(); i++) { for (int j = 1; j <= numRegionReplicas; j++) { hRegionInfos.add(RegionReplicaUtil.getRegionInfoForReplica(regions.get(i), j)); } } hRegionInfos.addAll(regions); return hRegionInfos; }
/** * Add the specified set of regions to the hbase:meta table. */ protected static void addRegionsToMeta(final MasterProcedureEnv env, final HTableDescriptor hTableDescriptor, final List<HRegionInfo> regionInfos) throws IOException { MetaTableAccessor.addRegionsToMeta(env.getMasterServices().getConnection(), regionInfos, hTableDescriptor.getRegionReplication()); }
private int getRegionReplication(HRegionInfo r) throws IOException { if (tableStateManager != null) { HTableDescriptor htd = ((MasterServices)server).getTableDescriptors().get(r.getTable()); if (htd != null) { return htd.getRegionReplication(); } } return 1; }
/** * Get a list of replica regions that are: * not recorded in meta yet. We might not have recorded the locations * for the replicas since the replicas may not have been online yet, master restarted * in the middle of assigning, ZK erased, etc. * @param regionsRecordedInMeta the list of regions we know are recorded in meta * either as a default, or, as the location of a replica * @param master * @return list of replica regions * @throws IOException */ public static List<HRegionInfo> replicaRegionsNotRecordedInMeta( Set<HRegionInfo> regionsRecordedInMeta, MasterServices master)throws IOException { List<HRegionInfo> regionsNotRecordedInMeta = new ArrayList<HRegionInfo>(); for (HRegionInfo hri : regionsRecordedInMeta) { TableName table = hri.getTable(); HTableDescriptor htd = master.getTableDescriptors().get(table); // look at the HTD for the replica count. That's the source of truth int desiredRegionReplication = htd.getRegionReplication(); for (int i = 0; i < desiredRegionReplication; i++) { HRegionInfo replica = RegionReplicaUtil.getRegionInfoForReplica(hri, i); if (regionsRecordedInMeta.contains(replica)) continue; regionsNotRecordedInMeta.add(replica); } } return regionsNotRecordedInMeta; }
protected static List<HRegionInfo> addTableToMeta(final MasterProcedureEnv env, final HTableDescriptor hTableDescriptor, final List<HRegionInfo> regions) throws IOException { if (regions != null && regions.size() > 0) { ProcedureSyncWait.waitMetaRegions(env); // Add regions to META addRegionsToMeta(env, hTableDescriptor, regions); // Add replicas if needed List<HRegionInfo> newRegions = addReplicas(env, hTableDescriptor, regions); // Setup replication for region replicas if needed if (hTableDescriptor.getRegionReplication() > 1) { ServerRegionReplicaUtil.setupRegionReplicaReplication(env.getMasterConfiguration()); } return newRegions; } return regions; }
@Test public void testMetaHTDReplicaCount() throws Exception { assertTrue(TEST_UTIL.getAdmin().getTableDescriptor(TableName.META_TABLE_NAME) .getRegionReplication() == 3); }
private void undeployRegions(HbckInfo hi) throws IOException, InterruptedException { undeployRegionsForHbi(hi); // undeploy replicas of the region (but only if the method is invoked for the primary) if (hi.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) { return; } int numReplicas = admin.getTableDescriptor(hi.getTableName()).getRegionReplication(); for (int i = 1; i < numReplicas; i++) { if (hi.getPrimaryHRIForDeployedReplica() == null) continue; HRegionInfo hri = RegionReplicaUtil.getRegionInfoForReplica( hi.getPrimaryHRIForDeployedReplica(), i); HbckInfo h = regionInfoMap.get(hri.getEncodedName()); if (h != null) { undeployRegionsForHbi(h); //set skip checks; we undeployed it, and we don't want to evaluate this anymore //in consistency checks h.setSkipChecks(true); } } }
private AtlasEntity getTableEntity(String nameSpace, String tableName, String owner, AtlasEntity nameSpaceEntity, HTableDescriptor htd, AtlasEntity atlasEntity) { AtlasEntity ret = null; if (atlasEntity == null) { ret = new AtlasEntity(HBaseDataTypes.HBASE_TABLE.getName()); } else { ret = atlasEntity; } String tableQualifiedName = getTableQualifiedName(clusterName, nameSpace, tableName); ret.setAttribute(QUALIFIED_NAME, tableQualifiedName); ret.setAttribute(CLUSTERNAME, clusterName); ret.setAttribute(NAMESPACE, AtlasTypeUtil.getAtlasObjectId(nameSpaceEntity)); ret.setAttribute(NAME, tableName); ret.setAttribute(DESCRIPTION_ATTR, tableName); ret.setAttribute(OWNER, owner); ret.setAttribute(URI, tableName); ret.setAttribute(ATTR_TABLE_MAX_FILESIZE, htd.getMaxFileSize()); ret.setAttribute(ATTR_TABLE_REPLICATION_PER_REGION, htd.getRegionReplication()); ret.setAttribute(ATTR_TABLE_ISREADONLY, htd.isReadOnly()); ret.setAttribute(ATTR_TABLE_ISCOMPACTION_ENABLED, htd.isCompactionEnabled()); ret.setAttribute(ATTR_TABLE_DURABLILITY, (htd.getDurability() != null ? htd.getDurability().name() : null)); return ret; }
private AtlasEntity getTableEntity(String nameSpace, String tableName, String owner, AtlasEntity nameSpaceEntity, HTableDescriptor htd, AtlasEntity atlasEntity) { AtlasEntity ret = null; if (atlasEntity == null) { ret = new AtlasEntity(HBaseDataTypes.HBASE_TABLE.getName()); } else { ret = atlasEntity; } String tableQualifiedName = getTableQualifiedName(clusterName, nameSpace, tableName); ret.setAttribute(QUALIFIED_NAME, tableQualifiedName); ret.setAttribute(CLUSTERNAME, clusterName); ret.setAttribute(NAMESPACE, AtlasTypeUtil.getAtlasObjectId(nameSpaceEntity)); ret.setAttribute(NAME, tableName); ret.setAttribute(DESCRIPTION_ATTR, tableName); ret.setAttribute(OWNER, owner); ret.setAttribute(URI, tableName); ret.setAttribute(ATTR_TABLE_MAX_FILESIZE, htd.getMaxFileSize()); ret.setAttribute(ATTR_TABLE_REPLICATION_PER_REGION, htd.getRegionReplication()); ret.setAttribute(ATTR_TABLE_ISREADONLY, htd.isReadOnly()); ret.setAttribute(ATTR_TABLE_ISCOMPACTION_ENABLED, htd.isCompactionEnabled()); ret.setAttribute(ATTR_TABLE_DURABLILITY, (htd.getDurability() != null ? htd.getDurability().name() : null)); return ret; }