@VisibleForTesting protected void createDirForRemoteWAL(MasterProcedureEnv env) throws IOException { MasterFileSystem mfs = env.getMasterFileSystem(); Path remoteWALDir = new Path(mfs.getWALRootDir(), ReplicationUtils.REMOTE_WAL_DIR_NAME); Path remoteWALDirForPeer = ReplicationUtils.getPeerRemoteWALDir(remoteWALDir, peerId); FileSystem walFs = mfs.getWALFileSystem(); if (walFs.exists(remoteWALDirForPeer)) { LOG.warn("Wal dir {} already exists, usually this should not happen, continue anyway", remoteWALDirForPeer); } else if (!walFs.mkdirs(remoteWALDirForPeer)) { throw new IOException("Failed to create remote wal dir " + remoteWALDirForPeer); } }
protected final Path getRemoteWALDir(MasterFileSystem mfs, String peerId) { Path remoteWALDir = new Path(mfs.getWALRootDir(), ReplicationUtils.REMOTE_WAL_DIR_NAME); return getRemoteWALDir(remoteWALDir, peerId); }
public SplitWALManager(MasterServices master) { this.master = master; this.conf = master.getConfiguration(); this.splitWorkerAssigner = new SplitWorkerAssigner(this.master, conf.getInt(HBASE_SPLIT_WAL_MAX_SPLITTER, DEFAULT_HBASE_SPLIT_WAL_MAX_SPLITTER)); this.rootDir = master.getMasterFileSystem().getWALRootDir(); this.fs = master.getMasterFileSystem().getFileSystem(); }
public MasterWalManager(MasterServices services) throws IOException { this(services.getConfiguration(), services.getMasterFileSystem().getWALFileSystem(), services.getMasterFileSystem().getWALRootDir(), services); }
public SyncReplicationReplayWALManager(MasterServices services) throws IOException, ReplicationException { this.serverManager = services.getServerManager(); this.fs = services.getMasterFileSystem().getWALFileSystem(); this.walRootDir = services.getMasterFileSystem().getWALRootDir(); this.remoteWALDir = new Path(this.walRootDir, ReplicationUtils.REMOTE_WAL_DIR_NAME); serverManager.registerListener(new ServerListener() { @Override public void serverAdded(ServerName serverName) { MasterProcedureScheduler scheduler = services.getMasterProcedureExecutor().getEnvironment().getProcedureScheduler(); for (UsedReplayWorkersForPeer usedWorkers : usedWorkersByPeer.values()) { synchronized (usedWorkers) { usedWorkers.wake(scheduler); } } } }); }
when(mfs.getWALRootDir()).thenReturn(new Path("/"));
@Test public void testCleanupMetaWAL() throws Exception { TEST_UTIL.createTable(TableName.valueOf("test"), "cf"); HRegionServer serverWithMeta = TEST_UTIL.getMiniHBaseCluster() .getRegionServer(TEST_UTIL.getMiniHBaseCluster().getServerWithMeta()); TEST_UTIL.getAdmin() .move(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), null); LOG.info("KILL"); TEST_UTIL.getMiniHBaseCluster().killRegionServer(serverWithMeta.getServerName()); LOG.info("WAIT"); TEST_UTIL.waitFor(30000, () -> TEST_UTIL.getMiniHBaseCluster().getMaster().getProcedures().stream() .filter(p -> p instanceof ServerCrashProcedure && p.isFinished()).count() > 0); LOG.info("DONE WAITING"); MasterFileSystem fs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); Path walPath = new Path(fs.getWALRootDir(), HConstants.HREGION_LOGDIR_NAME); for (FileStatus status : FSUtils.listStatus(fs.getFileSystem(), walPath)) { if (status.getPath().toString().contains(SPLITTING_EXT)) { fail("Should not have splitting wal dir here:" + status); } } } }
@Test public void testFsUriSetProperly() throws Exception { HMaster master = UTIL.getMiniHBaseCluster().getMaster(); MasterFileSystem fs = master.getMasterFileSystem(); Path masterRoot = FSUtils.getRootDir(fs.getConfiguration()); Path rootDir = FSUtils.getRootDir(fs.getFileSystem().getConf()); assertEquals(masterRoot, rootDir); assertEquals(FSUtils.getWALRootDir(UTIL.getConfiguration()), fs.getWALRootDir()); } }
log.rollWriter(); String walInputDir = new Path(cluster.getMaster().getMasterFileSystem() .getWALRootDir(), HConstants.HREGION_LOGDIR_NAME).toString();
FileSystem fs2 = UTIL2.getTestFileSystem(); REMOTE_WAL_DIR1 = new Path(UTIL1.getMiniHBaseCluster().getMaster().getMasterFileSystem().getWALRootDir(), "remoteWALs").makeQualified(fs1.getUri(), fs1.getWorkingDirectory()); REMOTE_WAL_DIR2 = new Path(UTIL2.getMiniHBaseCluster().getMaster().getMasterFileSystem().getWALRootDir(), "remoteWALs").makeQualified(fs2.getUri(), fs2.getWorkingDirectory()); UTIL1.getAdmin().addReplicationPeer(PEER_ID,
new Path(mfs.getWALRootDir(), ReplicationUtils.REMOTE_WAL_DIR_NAME), PEER_ID); FileStatus[] remoteWALStatus = mfs.getWALFileSystem().listStatus(remoteWALDir); assertEquals(1, remoteWALStatus.length);
new Path(mfs.getWALRootDir(), ReplicationUtils.REMOTE_WAL_DIR_NAME), PEER_ID); FileStatus[] remoteWALStatus = mfs.getWALFileSystem().listStatus(remoteWALDir); assertEquals(1, remoteWALStatus.length);
@Before public void before() throws IOException { MasterFileSystem mfs = Mockito.mock(MasterFileSystem.class); Mockito.when(mfs.getWALFileSystem()).thenReturn(HTU.getTestFileSystem()); Path walRootDir = HTU.createWALRootDir(); Mockito.when(mfs.getWALRootDir()).thenReturn(walRootDir); this.masterServices = Mockito.mock(MasterServices.class); Mockito.when(this.masterServices.getConfiguration()).thenReturn(HTU.getConfiguration()); Mockito.when(this.masterServices.getMasterFileSystem()).thenReturn(mfs); Mockito.when(this.masterServices.getServerName()). thenReturn(ServerName.parseServerName("master.example.org,0123,456")); this.mwm = new MasterWalManager(this.masterServices); }
@Test public void testCleanupMetaWAL() throws Exception { TEST_UTIL.createTable(TableName.valueOf("test"), "cf"); HRegionServer serverWithMeta = TEST_UTIL.getMiniHBaseCluster() .getRegionServer(TEST_UTIL.getMiniHBaseCluster().getServerWithMeta()); TEST_UTIL.getAdmin() .move(RegionInfoBuilder.FIRST_META_REGIONINFO.getEncodedNameAsBytes(), null); LOG.info("KILL"); TEST_UTIL.getMiniHBaseCluster().killRegionServer(serverWithMeta.getServerName()); LOG.info("WAIT"); TEST_UTIL.waitFor(30000, () -> TEST_UTIL.getMiniHBaseCluster().getMaster().getProcedures().stream() .filter(p -> p instanceof ServerCrashProcedure && p.isFinished()).count() > 0); LOG.info("DONE WAITING"); MasterFileSystem fs = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterFileSystem(); Path walPath = new Path(fs.getWALRootDir(), HConstants.HREGION_LOGDIR_NAME); for (FileStatus status : FSUtils.listStatus(fs.getFileSystem(), walPath)) { if (status.getPath().toString().contains(SPLITTING_EXT)) { fail("Should not have splitting wal dir here:" + status); } } } }
log.rollWriter(); String walInputDir = new Path(cluster.getMaster().getMasterFileSystem() .getWALRootDir(), HConstants.HREGION_LOGDIR_NAME).toString();
log.rollWriter(); String walInputDir = new Path(cluster.getMaster().getMasterFileSystem() .getWALRootDir(), HConstants.HREGION_LOGDIR_NAME).toString();
log.rollWriter(); String walInputDir = new Path(cluster.getMaster().getMasterFileSystem() .getWALRootDir(), HConstants.HREGION_LOGDIR_NAME).toString();
log.rollWriter(); String walInputDir = new Path(cluster.getMaster().getMasterFileSystem() .getWALRootDir(), HConstants.HREGION_LOGDIR_NAME).toString();
@Test public void testFsUriSetProperly() throws Exception { HMaster master = UTIL.getMiniHBaseCluster().getMaster(); MasterFileSystem fs = master.getMasterFileSystem(); Path masterRoot = FSUtils.getRootDir(fs.getConfiguration()); Path rootDir = FSUtils.getRootDir(fs.getFileSystem().getConf()); assertEquals(masterRoot, rootDir); assertEquals(FSUtils.getWALRootDir(UTIL.getConfiguration()), fs.getWALRootDir()); } }