public void dumpSidelinedRegions(Map<Path, HbckInfo> regions) { for (Map.Entry<Path, HbckInfo> entry: regions.entrySet()) { TableName tableName = entry.getValue().getTableName(); Path path = entry.getKey(); errors.print("This sidelined region dir should be bulk loaded: " + path.toString()); errors.print("Bulk load command looks like: " + "hbase org.apache.hadoop.hbase.tool.LoadIncrementalHFiles " + path.toUri().getPath() + " "+ tableName); } }
/** * ls -r for debugging purposes */ public static void debugLsr(Configuration conf, Path p, ErrorReporter errors) throws IOException { if (!LOG.isDebugEnabled() || p == null) { return; } FileSystem fs = p.getFileSystem(conf); if (!fs.exists(p)) { // nothing return; } errors.print(p.toString()); if (fs.isFile(p)) { return; } if (fs.getFileStatus(p).isDirectory()) { FileStatus[] fss= fs.listStatus(p); for (FileStatus status : fss) { debugLsr(conf, status.getPath(), errors); } } } }
synchronized (hbi) { if (hbi.getHdfsRegionDir() != null) { errors.print("Directory " + encodedName + " duplicate??" + hbi.getHdfsRegionDir());
errors.print("---- Table '" + this.tableName + "': region split map"); dump(splits, regions); errors.print("---- Table '" + this.tableName + "': overlap groups"); dumpOverlapProblems(overlapGroups); errors.print("There are " + overlapGroups.keySet().size() + " overlap groups with " + overlapGroups.size() + " overlapping regions"); errors.print("---- Table '" + this.tableName + "': sidelined big overlapped regions"); dumpSidelinedRegions(sidelinedRegions);
errors.print(""); return true;
errors.print("");
metaHbckInfo.getReplicaId() + " is found on more than one region."); if (shouldFixAssignments()) { errors.print("Trying to fix a problem with hbase:meta, replicaId " + metaHbckInfo.getReplicaId() +".."); setShouldRerun(); ", deployed " + metaRegions.size()); if (shouldFixAssignments()) { errors.print("Trying to undeploy excess replica, replicaId: " + entry.getKey() + " of hbase:meta.."); setShouldRerun();
errors.print("");
Map<String, Path> allFiles = FSUtils.getTableStoreFilePathMap(fs, hbaseRoot, new FSUtils.ReferenceFileFilter(fs), executor, errors); errors.print(""); LOG.info("Validating mapping using HDFS state"); for (Path path: allFiles.values()) {
out.print("Checked " + hfilesChecked.get() + " hfile for corruption"); out.print(" HFiles corrupted: " + corrupted.size()); if (inQuarantineMode) { out.print(" HFiles successfully quarantined: " + quarantined.size()); for (Path sq : quarantined) { out.print(" " + sq); out.print(" HFiles failed quarantine: " + failures.size()); for (Path fq : failures) { out.print(" " + fq); out.print(" HFiles moved while checking: " + missing.size()); for (Path mq : missing) { out.print(" " + mq); out.print(" Mob files successfully quarantined: " + quarantinedMobFiles.size()); for (Path sq : quarantinedMobFiles) { out.print(" " + sq); out.print(" Mob files failed quarantine: " + failureMobFiles.size()); for (Path fq : failureMobFiles) { out.print(" " + fq); out.print(" Mob files moved while checking: " + missedMobFiles.size()); for (Path mq : missedMobFiles) { out.print(" " + mq); out.print("Summary: " + initialState + " => " + fixedState); out.print("Mob summary: " + initialMobState + " => " + fixedMobState);
Map<String, Path> allFiles = FSUtils .getTableStoreFilePathMap(fs, hbaseRoot, new FSUtils.HFileLinkFilter(), executor, errors); errors.print("");
errors.print("Number of live region servers: " + regionServers.size()); if (details) { for (ServerName rsinfo: regionServers) { errors.print(" " + rsinfo.getServerName()); errors.print("Number of dead region servers: " + deadRegionServers.size()); if (details) { for (ServerName name: deadRegionServers) { errors.print(" " + name); errors.print("Master: " + status.getMasterName()); errors.print("Number of backup masters: " + backupMasters.size()); if (details) { for (ServerName name: backupMasters) { errors.print(" " + name); errors.print("Average load: " + status.getAverageLoad()); errors.print("Number of requests: " + status.getRequestCount()); errors.print("Number of regions: " + status.getRegionCount()); errors.print("Number of regions in transition: " + rits.size()); if (details) { for (RegionState state: rits) { errors.print(" " + state.toDescriptiveString());
throws IOException, KeeperException, InterruptedException, ReplicationException { errors.print("Version: " + status.getHBaseVersion());
StringBuilder sb = new StringBuilder(); int numOfSkippedRegions; errors.print("Summary:"); for (TableInfo tInfo : tablesInfo.values()) { numOfSkippedRegions = (skippedRegions.containsKey(tInfo.getName())) ? errors.print("Table " + tInfo.getName() + " is inconsistent."); } else if (numOfSkippedRegions > 0){ errors.print("Table " + tInfo.getName() + " is okay (with " + numOfSkippedRegions + " skipped regions)."); errors.print("Table " + tInfo.getName() + " is okay."); errors.print(" Number of regions: " + tInfo.getNumRegions()); if (numOfSkippedRegions > 0) { Set<String> skippedRegionStrings = skippedRegions.get(tInfo.getName()); sb.append(" " + server.toString()); errors.print(sb.toString());
private void tryAssignmentRepair(HbckInfo hbi, String msg) throws IOException, KeeperException, InterruptedException { // If we are trying to fix the errors if (shouldFixAssignments()) { errors.print(msg); undeployRegions(hbi); setShouldRerun(); RegionInfo hri = hbi.getHdfsHRI(); if (hri == null) { hri = hbi.metaEntry; } HBaseFsckRepair.fixUnassigned(admin, hri); HBaseFsckRepair.waitUntilAssigned(admin, hri); // also assign replicas if needed (do it only when this call operates on a primary replica) if (hbi.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) return; int replicationCount = admin.getTableDescriptor(hri.getTable()).getRegionReplication(); for (int i = 1; i < replicationCount; i++) { hri = RegionReplicaUtil.getRegionInfoForReplica(hri, i); HbckInfo h = regionInfoMap.get(hri.getEncodedName()); if (h != null) { undeployRegions(h); //set skip checks; we undeploy & deploy it; we don't want to evaluate this hbi anymore //in consistency checks h.setSkipChecks(true); } HBaseFsckRepair.fixUnassigned(admin, hri); HBaseFsckRepair.waitUntilAssigned(admin, hri); } } }
/** * This dumps data in a visually reasonable way for visual debugging * * @param splits * @param regions */ void dump(SortedSet<byte[]> splits, Multimap<byte[], HbckInfo> regions) { // we display this way because the last end key should be displayed as well. StringBuilder sb = new StringBuilder(); for (byte[] k : splits) { sb.setLength(0); // clear out existing buffer, if any. sb.append(Bytes.toStringBinary(k) + ":\t"); for (HbckInfo r : regions.get(k)) { sb.append("[ "+ r.toString() + ", " + Bytes.toStringBinary(r.getEndKey())+ "]\t"); } errors.print(sb.toString()); } } }
/** * TODO -- need to add tests for this. */ private void reportTablesInFlux() { AtomicInteger numSkipped = new AtomicInteger(0); TableDescriptor[] allTables = getTables(numSkipped); errors.print("Number of Tables: " + allTables.length); if (details) { if (numSkipped.get() > 0) { errors.detail("Number of Tables in flux: " + numSkipped.get()); } for (TableDescriptor td : allTables) { errors.detail(" Table: " + td.getTableName() + "\t" + (td.isReadOnly() ? "ro" : "rw") + "\t" + (td.isMetaRegion() ? "META" : " ") + "\t" + " families: " + td.getColumnFamilyCount()); } } }
private void assignMetaReplica(int replicaId) throws IOException, KeeperException, InterruptedException { errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta, replicaId " + replicaId +" is not found on any region."); if (shouldFixAssignments()) { errors.print("Trying to fix a problem with hbase:meta.."); setShouldRerun(); // try to fix it (treat it as unassigned region) RegionInfo h = RegionReplicaUtil.getRegionInfoForReplica( RegionInfoBuilder.FIRST_META_REGIONINFO, replicaId); HBaseFsckRepair.fixUnassigned(admin, h); HBaseFsckRepair.waitUntilAssigned(admin, h); } }
public void dumpOverlapProblems(Multimap<byte[], HbckInfo> regions) { // we display this way because the last end key should be displayed as // well. for (byte[] k : regions.keySet()) { errors.print(Bytes.toStringBinary(k) + ":"); for (HbckInfo r : regions.get(k)) { errors.print("[ " + r.toString() + ", " + Bytes.toStringBinary(r.getEndKey()) + "]"); } errors.print("----"); } }