public void checkUnDeletedQueues() throws ReplicationException { undeletedQueueIds = getUnDeletedQueues(); undeletedQueueIds.forEach((replicator, queueIds) -> { queueIds.forEach(queueId -> { ReplicationQueueInfo queueInfo = new ReplicationQueueInfo(queueId); String msg = "Undeleted replication queue for removed peer found: " + String.format("[removedPeerId=%s, replicator=%s, queueId=%s]", queueInfo.getPeerId(), replicator, queueId); errorReporter.reportError(HBaseFsck.ErrorReporter.ERROR_CODE.UNDELETED_REPLICATION_QUEUE, msg); }); }); undeletedHFileRefsPeerIds = getUndeletedHFileRefsPeers(); undeletedHFileRefsPeerIds.stream() .map( peerId -> "Undeleted replication hfile-refs queue for removed peer " + peerId + " found") .forEach(msg -> errorReporter .reportError(HBaseFsck.ErrorReporter.ERROR_CODE.UNDELETED_REPLICATION_QUEUE, msg)); }
errors.reportError(ERROR_CODE.LINGERING_HFILELINK, "Found lingering HFileLink " + path); if (!shouldFixHFileLinks()) continue;
private void preCheckPermission() throws IOException, AccessDeniedException { if (shouldIgnorePreCheckPermission()) { return; } Path hbaseDir = FSUtils.getRootDir(getConf()); FileSystem fs = hbaseDir.getFileSystem(getConf()); UserProvider userProvider = UserProvider.instantiate(getConf()); UserGroupInformation ugi = userProvider.getCurrent().getUGI(); FileStatus[] files = fs.listStatus(hbaseDir); for (FileStatus file : files) { try { FSUtils.checkAccess(ugi, file, FsAction.WRITE); } catch (AccessDeniedException ace) { LOG.warn("Got AccessDeniedException when preCheckPermission ", ace); errors.reportError(ERROR_CODE.WRONG_USAGE, "Current user " + ugi.getUserName() + " does not have write perms to " + file.getPath() + ". Please rerun hbck as hdfs user " + file.getOwner()); throw ace; } } }
TableState state = MetaTableAccessor.getTableState(connection, tableName); if (state != null) { errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE, tableName + " unable to delete dangling table state " + tableState); errors.reportError(ERROR_CODE.ORPHAN_TABLE_STATE, tableName + " has dangling table state " + tableState); TableState newState = MetaTableAccessor.getTableState(connection, tableName); if (newState == null) { errors.reportError(ERROR_CODE.NO_TABLE_STATE, "Unable to change state for table " + tableName + " in meta "); errors.reportError(ERROR_CODE.NO_TABLE_STATE, tableName + " has no state in meta ");
HConstants.EMPTY_START_ROW, false, false); if (rl == null) { errors.reportError(ERROR_CODE.NULL_META_REGION, "META region was not found in ZooKeeper"); return false; errors.reportError(ERROR_CODE.NULL_META_REGION, "META region location is null"); return false; errors.reportError(ERROR_CODE.NULL_META_REGION, "META location regionInfo is null"); return false; errors.reportError(ERROR_CODE.NULL_META_REGION, "META location hostName is null"); return false;
errors.reportError(ERROR_CODE.RS_CONNECT_FAILURE, "RegionServer: " + rsinfo.getServerName() + " Unable to fetch region information. " + e); throw e;
@Override public synchronized Void call() throws IOException { // only load entries that haven't been loaded yet. if (hbi.getHdfsHRI() == null) { try { errors.progress(); hbck.loadHdfsRegioninfo(hbi); } catch (IOException ioe) { String msg = "Orphan region in HDFS: Unable to load .regioninfo from table " + hbi.getTableName() + " in hdfs dir " + hbi.getHdfsRegionDir() + "! It may be an invalid format or version file. Treating as " + "an orphaned regiondir."; errors.reportError(ERROR_CODE.ORPHAN_HDFS_REGION, msg); try { hbck.debugLsr(hbi.getHdfsRegionDir()); } catch (IOException ioe2) { LOG.error("Unable to read directory " + hbi.getHdfsRegionDir(), ioe2); throw ioe2; } hbck.orphanHdfsDirs.add(hbi); throw ioe; } } return null; } }
/** * This is a special case hole -- when the first region of a table is * missing from META, HBase doesn't acknowledge the existance of the * table. */ @Override public void handleRegionStartKeyNotEmpty(HbckInfo next) throws IOException { errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY, "First region should start with an empty key. Creating a new " + "region and regioninfo in HDFS to plug the hole.", getTableInfo(), next); TableDescriptor htd = getTableInfo().getHTD(); // from special EMPTY_START_ROW to next region's startKey RegionInfo newRegion = RegionInfoBuilder.newBuilder(htd.getTableName()) .setStartKey(HConstants.EMPTY_START_ROW) .setEndKey(next.getStartKey()) .build(); // TODO test HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd); LOG.info("Table region start key was not empty. Created new empty region: " + newRegion + " " +region); fixes++; }
/** * There is a hole in the hdfs regions that violates the table integrity * rules. Create a new empty region that patches the hole. */ @Override public void handleHoleInRegionChain(byte[] holeStartKey, byte[] holeStopKey) throws IOException { errors.reportError( ERROR_CODE.HOLE_IN_REGION_CHAIN, "There is a hole in the region chain between " + Bytes.toStringBinary(holeStartKey) + " and " + Bytes.toStringBinary(holeStopKey) + ". Creating a new regioninfo and region " + "dir in hdfs to plug the hole."); TableDescriptor htd = getTableInfo().getHTD(); RegionInfo newRegion = RegionInfoBuilder.newBuilder(htd.getTableName()) .setStartKey(holeStartKey) .setEndKey(holeStopKey) .build(); HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd); LOG.info("Plugged hole by creating new empty region: "+ newRegion + " " +region); fixes++; }
public void addRegionInfo(HbckInfo hir) { if (Bytes.equals(hir.getEndKey(), HConstants.EMPTY_END_ROW)) { // end key is absolute end key, just add it. // ignore replicas other than primary for these checks if (hir.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) sc.add(hir); return; } // if not the absolute end key, check for cycle if (Bytes.compareTo(hir.getStartKey(), hir.getEndKey()) > 0) { errors.reportError( ERROR_CODE.REGION_CYCLE, String.format("The endkey for this region comes before the " + "startkey, startkey=%s, endkey=%s", Bytes.toStringBinary(hir.getStartKey()), Bytes.toStringBinary(hir.getEndKey())), this, hir); backwards.add(hir); return; } // main case, add to split calculator // ignore replicas other than primary for these checks if (hir.getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID) sc.add(hir); }
private boolean isOptionsSupported(String[] args) { boolean result = true; String hbaseServerVersion = status.getHBaseVersion(); if (VersionInfo.compareVersion("2.any.any", hbaseServerVersion) < 0) { // Process command-line args. for (String arg : args) { if (unsupportedOptionsInV2.contains(arg)) { errors.reportError(ERROR_CODE.UNSUPPORTED_OPTION, "option '" + arg + "' is not " + "supportted!"); result = false; break; } } } return result; }
@Override public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException { errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY, "Last region should end with an empty key. Creating a new " + "region and regioninfo in HDFS to plug the hole.", getTableInfo()); TableDescriptor htd = getTableInfo().getHTD(); // from curEndKey to EMPTY_START_ROW RegionInfo newRegion = RegionInfoBuilder.newBuilder(htd.getTableName()) .setStartKey(curEndKey) .setEndKey(HConstants.EMPTY_START_ROW) .build(); HRegion region = HBaseFsckRepair.createHDFSRegionDir(conf, newRegion, htd); LOG.info("Table region end key was not empty. Created new empty region: " + newRegion + " " + region); fixes++; }
private void assignMetaReplica(int replicaId) throws IOException, KeeperException, InterruptedException { errors.reportError(ERROR_CODE.NO_META_REGION, "hbase:meta, replicaId " + replicaId +" is not found on any region."); if (shouldFixAssignments()) { errors.print("Trying to fix a problem with hbase:meta.."); setShouldRerun(); // try to fix it (treat it as unassigned region) RegionInfo h = RegionReplicaUtil.getRegionInfoForReplica( RegionInfoBuilder.FIRST_META_REGIONINFO, replicaId); HBaseFsckRepair.fixUnassigned(admin, h); HBaseFsckRepair.waitUntilAssigned(admin, h); } }
@Override public void handleDuplicateStartKeys(HbckInfo r1, HbckInfo r2) throws IOException{ byte[] key = r1.getStartKey(); // dup start key errors.reportError(ERROR_CODE.DUPE_STARTKEYS, "Multiple regions have the same startkey: " + Bytes.toStringBinary(key), getTableInfo(), r1); errors.reportError(ERROR_CODE.DUPE_STARTKEYS, "Multiple regions have the same startkey: " + Bytes.toStringBinary(key), getTableInfo(), r2); }
@Override public void handleRegionStartKeyNotEmpty(HbckInfo hi) throws IOException{ errors.reportError(ERROR_CODE.FIRST_REGION_STARTKEY_NOT_EMPTY, "First region should start with an empty key. You need to " + " create a new region and regioninfo in HDFS to plug the hole.", getTableInfo(), hi); }
@Override public void handleDegenerateRegion(HbckInfo hi) throws IOException{ errors.reportError(ERROR_CODE.DEGENERATE_REGION, "Region has the same start and end key.", getTableInfo(), hi); }
@Override public void handleRegionEndKeyNotEmpty(byte[] curEndKey) throws IOException { errors.reportError(ERROR_CODE.LAST_REGION_ENDKEY_NOT_EMPTY, "Last region should end with an empty key. You need to " + "create a new region and regioninfo in HDFS to plug the hole.", getTableInfo()); }
@Override public void handleHoleInRegionChain(byte[] holeStart, byte[] holeStop) throws IOException{ errors.reportError( ERROR_CODE.HOLE_IN_REGION_CHAIN, "There is a hole in the region chain between " + Bytes.toStringBinary(holeStart) + " and " + Bytes.toStringBinary(holeStop) + ". You need to create a new .regioninfo and region " + "dir in hdfs to plug the hole."); } }
@Override public void handleOverlapInRegionChain(HbckInfo hi1, HbckInfo hi2) throws IOException{ errors.reportError(ERROR_CODE.OVERLAP_IN_REGION_CHAIN, "There is an overlap in the region chain.", getTableInfo(), hi1, hi2); }
@Override public void handleSplit(HbckInfo r1, HbckInfo r2) throws IOException{ byte[] key = r1.getStartKey(); // dup start key errors.reportError(ERROR_CODE.DUPE_ENDKEYS, "Multiple regions have the same regionID: " + Bytes.toStringBinary(key), getTableInfo(), r1); errors.reportError(ERROR_CODE.DUPE_ENDKEYS, "Multiple regions have the same regionID: " + Bytes.toStringBinary(key), getTableInfo(), r2); }