@Override public List<Procedure<?>> getProcedures() throws IOException { if (cpHost != null) { cpHost.preGetProcedures(); } @SuppressWarnings({ "unchecked", "rawtypes" }) List<Procedure<?>> procList = (List) this.procedureExecutor.getProcedures(); if (cpHost != null) { cpHost.postGetProcedures(procList); } return procList; }
protected byte[] waitOnFuture(final Future<byte[]> future) throws Exception { try { return future.get(3, TimeUnit.MINUTES); } catch (ExecutionException e) { LOG.info("ExecutionException", e); Exception ee = (Exception) e.getCause(); if (ee instanceof InterruptedIOException) { for (Procedure<?> p : this.master.getMasterProcedureExecutor().getProcedures()) { LOG.info(p.toStringDetails()); } } throw (Exception) e.getCause(); } }
private long getSCPProcId(ProcedureExecutor<?> procExec) { util.waitFor(30000, () -> !procExec.getProcedures().isEmpty()); return procExec.getActiveProcIds().stream().mapToLong(Long::longValue).min().getAsLong(); }
private boolean shouldSubmitSCP(ServerName serverName) { // check if there is already a SCP of this server running List<Procedure<MasterProcedureEnv>> procedures = master.getMasterProcedureExecutor().getProcedures(); for (Procedure<MasterProcedureEnv> procedure : procedures) { if (procedure instanceof ServerCrashProcedure) { if (serverName.compareTo(((ServerCrashProcedure) procedure).getServerName()) == 0 && !procedure.isFinished()) { LOG.info("there is already a SCP of this server {} running, pid {}", serverName, procedure.getProcId()); return false; } } } return true; } }
@Test public void testBypassingProcedureWithParentRecursive() throws Exception { final RootProcedure proc = new RootProcedure(); long rootId = procExecutor.submitProcedure(proc); htu.waitFor(5000, () -> procExecutor.getProcedures().stream() .filter(p -> p.getParentProcId() == rootId).collect(Collectors.toList()) .size() > 0); SuspendProcedure suspendProcedure = (SuspendProcedure)procExecutor.getProcedures().stream() .filter(p -> p.getParentProcId() == rootId).collect(Collectors.toList()).get(0); assertTrue(procExecutor.bypassProcedure(rootId, 1000, false, true)); htu.waitFor(5000, () -> proc.isSuccess() && proc.isBypass()); LOG.info("{} finished", proc); }
@Test public void testBypassingProcedureWithParent() throws Exception { final RootProcedure proc = new RootProcedure(); long rootId = procExecutor.submitProcedure(proc); htu.waitFor(5000, () -> procExecutor.getProcedures().stream() .filter(p -> p.getParentProcId() == rootId).collect(Collectors.toList()) .size() > 0); SuspendProcedure suspendProcedure = (SuspendProcedure)procExecutor.getProcedures().stream() .filter(p -> p.getParentProcId() == rootId).collect(Collectors.toList()).get(0); assertTrue(procExecutor.bypassProcedure(suspendProcedure.getProcId(), 1000, false, false)); htu.waitFor(5000, () -> proc.isSuccess() && proc.isBypass()); LOG.info("{} finished", proc); }
/** * @return True if region is online and scannable else false if an error or shutdown (Otherwise * we just block in here holding up all forward-progess). */ private boolean isRegionOnline(RegionInfo ri) throws InterruptedException { RetryCounter rc = null; while (!isStopped()) { RegionState rs = this.assignmentManager.getRegionStates().getRegionState(ri); if (rs.isOpened()) { if (this.getServerManager().isServerOnline(rs.getServerName())) { return true; } } // Region is not OPEN. Optional<Procedure<MasterProcedureEnv>> optProc = this.procedureExecutor.getProcedures(). stream().filter(p -> p instanceof ServerCrashProcedure).findAny(); // TODO: Add a page to refguide on how to do repair. Have this log message point to it. // Page will talk about loss of edits, how to schedule at least the meta WAL recovery, and // then how to assign including how to break region lock if one held. LOG.warn("{} is NOT online; state={}; ServerCrashProcedures={}. Master startup cannot " + "progress, in holding-pattern until region onlined.", ri.getRegionNameAsString(), rs, optProc.isPresent()); // Check once-a-minute. if (rc == null) { rc = new RetryCounterFactory(1000).create(); } Threads.sleep(rc.getBackoffTimeAndIncrementAttempts()); } return false; }
@Test public void testTableProcedureSubProcedureDeadLock() throws Exception { // the shared procedure will also schedule a shared procedure, but after the exclusive procedure long procId1 = procExec.submitProcedure(new TableShardParentProcedure()); long procId2 = procExec.submitProcedure(new TableExclusiveProcedure()); procExec.startWorkers(); UTIL.waitFor(10000, () -> procExec.getProcedures().stream().anyMatch(p -> p instanceof TableSharedProcedure)); procExec.getProcedures().stream().filter(p -> p instanceof TableSharedProcedure) .map(p -> (TableSharedProcedure) p).forEach(p -> p.latch.release()); ((TableExclusiveProcedure) procExec.getProcedure(procId2)).latch.release(); UTIL.waitFor(10000, () -> procExec.isFinished(procId1)); UTIL.waitFor(10000, () -> procExec.isFinished(procId2)); } }
@Test public void testProcedureShouldNotCleanOnLoad() throws Exception { createProcExecutor(); final RootProcedure proc = new RootProcedure(); long rootProc = procExecutor.submitProcedure(proc); LOG.info("Begin to execute " + rootProc); // wait until the child procedure arrival htu.waitFor(10000, () -> procExecutor.getProcedures().size() >= 2); SuspendProcedure suspendProcedure = (SuspendProcedure) procExecutor .getProcedures().get(1); // wait until the suspendProcedure executed suspendProcedure.latch.countDown(); Thread.sleep(100); // roll the procedure log LOG.info("Begin to roll log "); procStore.rollWriterForTesting(); LOG.info("finish to roll log "); Thread.sleep(500); LOG.info("begin to restart1 "); ProcedureTestingUtility.restart(procExecutor, true); LOG.info("finish to restart1 "); assertTrue(procExecutor.getProcedure(rootProc) != null); Thread.sleep(500); LOG.info("begin to restart2 "); ProcedureTestingUtility.restart(procExecutor, true); LOG.info("finish to restart2 "); assertTrue(procExecutor.getProcedure(rootProc) != null); }
@Test public void testFailCreateTable() throws Exception { conf.set(MASTER_COPROCESSOR_CONF_KEY, CreateFailObserver.class.getName()); TEST_UTIL.startMiniCluster(3); try { TEST_UTIL.createTable(TABLE, FAMILY); } catch (AccessDeniedException e) { LOG.debug("Ignoring exception: ", e); Thread.sleep(evictionDelay * 3); } List<Procedure<MasterProcedureEnv>> procedureInfos = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor().getProcedures(); for (Procedure procedureInfo : procedureInfos) { if (procedureInfo.getProcName().equals("CreateTableProcedure") && procedureInfo.getState() == ProcedureProtos.ProcedureState.ROLLEDBACK) { fail("Found procedure " + procedureInfo + " that hasn't been cleaned up"); } } }
@Test public void testFailCreateTableAction() throws Exception { conf.set(MASTER_COPROCESSOR_CONF_KEY, CreateFailObserverHandler.class.getName()); TEST_UTIL.startMiniCluster(3); try { TEST_UTIL.createTable(TABLE, FAMILY); fail("Table shouldn't be created"); } catch (AccessDeniedException e) { LOG.debug("Ignoring exception: ", e); Thread.sleep(evictionDelay * 3); } List<Procedure<MasterProcedureEnv>> procedureInfos = TEST_UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor().getProcedures(); for (Procedure procedureInfo : procedureInfos) { if (procedureInfo.getProcName().equals("CreateTableProcedure") && procedureInfo.getState() == ProcedureProtos.ProcedureState.ROLLEDBACK) { fail("Found procedure " + procedureInfo + " that hasn't been cleaned up"); } } }
am.moveAsync(new RegionPlan(region, rsn.getRegionLocation(), rsn.getRegionLocation())); TransitRegionStateProcedure proc = procExec.getProcedures().stream().filter(p -> p instanceof TransitRegionStateProcedure) .filter(p -> !p.isFinished()).map(p -> (TransitRegionStateProcedure) p).findAny().get();
UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor(); UTIL.waitFor(10000, () -> procExec.getProcedures().stream().filter(p -> !p.isFinished()) .filter(p -> p instanceof LockProcedure).map(p -> (LockProcedure) p) .filter(p -> NAME.equals(p.getTableName())).anyMatch(p -> !p.isLocked()));
@After public void tearDown() throws Exception { ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(procExec, false); // Kill all running procedures. for (Procedure<?> proc : procExec.getProcedures()) { procExec.abort(proc.getProcId()); ProcedureTestingUtility.waitProcedure(procExec, proc); } assertEquals(0, procExec.getEnvironment().getProcedureScheduler().size()); }
@Test public void test() throws Exception { TableDescriptor tableDescriptor = client.getDescriptor(); ProcedureExecutor<MasterProcedureEnv> executor = UTIL.getMiniHBaseCluster().getMaster() .getMasterProcedureExecutor(); MasterProcedureEnv env = executor.getEnvironment(); List<RegionInfo> regionInfos = admin.getRegions(TABLE_NAME); MergeTableRegionsProcedure mergeTableRegionsProcedure = new MergeTableRegionsProcedure( UTIL.getMiniHBaseCluster().getMaster().getMasterProcedureExecutor() .getEnvironment(), regionInfos.get(0), regionInfos.get(1)); ModifyTableProcedure modifyTableProcedure = new ModifyTableProcedure(env, tableDescriptor); long procModify = executor.submitProcedure(modifyTableProcedure); UTIL.waitFor(30000, () -> executor.getProcedures().stream() .filter(p -> p instanceof ModifyTableProcedure) .map(p -> (ModifyTableProcedure) p) .anyMatch(p -> TABLE_NAME.equals(p.getTableName()))); long proc = executor.submitProcedure(mergeTableRegionsProcedure); UTIL.waitFor(3000000, () -> UTIL.getMiniHBaseCluster().getMaster() .getMasterProcedureExecutor().isFinished(procModify)); Assert.assertEquals("Modify Table procedure should success!", ProcedureProtos.ProcedureState.SUCCESS, modifyTableProcedure.getState()); }
() -> executor.getProcedures().stream() .filter(p -> p instanceof TransitRegionStateProcedure) .map(p -> (TransitRegionStateProcedure) p)
() -> executor.getProcedures().stream() .filter(p -> p instanceof TransitRegionStateProcedure) .map(p -> (TransitRegionStateProcedure) p)
@Test public void testGetProcedures() throws Exception { final TableName tableName = TableName.valueOf(name.getMethodName()); final ProcedureExecutor<MasterProcedureEnv> procExec = TEST_UTIL.getHBaseCluster().getMaster().getMasterProcedureExecutor(); Procedure proc = new TestTableDDLProcedure(procExec.getEnvironment(), tableName); proc.setOwner(USER_OWNER); procExec.submitProcedure(proc); final List<Procedure<MasterProcedureEnv>> procList = procExec.getProcedures(); AccessTestAction getProceduresAction = new AccessTestAction() { @Override public Object run() throws Exception { ACCESS_CONTROLLER .postGetProcedures(ObserverContextImpl.createAndPrepare(CP_ENV)); return null; } }; verifyAllowed(getProceduresAction, SUPERUSER, USER_ADMIN, USER_GROUP_ADMIN); verifyAllowed(getProceduresAction, USER_OWNER); verifyIfNull( getProceduresAction, USER_RW, USER_RO, USER_NONE, USER_GROUP_READ, USER_GROUP_WRITE); }
@After public void tearDown() throws Exception { for (Procedure<?> proc : getMasterProcedureExecutor().getProcedures()) { if (proc instanceof LockProcedure) { ((LockProcedure) proc).unlock(getMasterProcedureExecutor().getEnvironment()); ProcedureTestingUtility.waitProcedure(getMasterProcedureExecutor(), proc); } } assertEquals(0, getMasterProcedureExecutor().getEnvironment().getProcedureScheduler().size()); }
List<Procedure<MasterProcedureEnv>> procedures = procExec.getProcedures(); assertTrue(procedures.size() >= 1); boolean found = false; ProcedureTestingUtility.waitNoProcedureRunning(procExec); ProcedureTestingUtility.assertProcNotFailed(procExec, procId); procedures = procExec.getProcedures(); for (Procedure proc: procedures) { assertTrue(proc.isSuccess());