public static void storeRestart(ProcedureStore procStore, ProcedureStore.ProcedureLoader loader) throws Exception { procStore.stop(false); procStore.start(procStore.getNumThreads()); procStore.recoverLease(); procStore.load(loader); }
private void updateStoreOnExec(RootProcedureState<TEnvironment> procStack, Procedure<TEnvironment> procedure, Procedure<TEnvironment>[] subprocs) { if (subprocs != null && !procedure.isFailed()) { if (LOG.isTraceEnabled()) { LOG.trace("Stored " + procedure + ", children " + Arrays.toString(subprocs)); } store.insert(procedure, subprocs); } else { LOG.trace("Store update {}", procedure); if (procedure.isFinished() && !procedure.hasParent()) { // remove child procedures final long[] childProcIds = procStack.getSubprocedureIds(); if (childProcIds != null) { store.delete(procedure, childProcIds); for (int i = 0; i < childProcIds.length; ++i) { procedures.remove(childProcIds[i]); } } else { store.update(procedure); } } else { store.update(procedure); } } }
final Iterator<Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>>> it = completed.entrySet().iterator(); while (it.hasNext() && store.isRunning()) { final Map.Entry<Long, CompletedProcedureRetainer<TEnvironment>> entry = it.next(); final CompletedProcedureRetainer<TEnvironment> retainer = entry.getValue(); batchIds[batchCount++] = entry.getKey(); if (batchCount == batchIds.length) { store.delete(batchIds, 0, batchCount); batchCount = 0; store.delete(batchIds, 0, batchCount);
@After public void tearDown() throws IOException { procExecutor.stop(); procStore.stop(false); fs.delete(logDir, true); }
@Before public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); Path testDir = htu.getDataTestDir(); fs = testDir.getFileSystem(htu.getConfiguration()); logDir = new Path(testDir, "proc-logs"); procEnv = new TestProcEnv(); procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), logDir); procExecutor = new ProcedureExecutor<>(htu.getConfiguration(), procEnv, procStore); procStore.start(1); ProcedureTestingUtility.initAndStartWorkers(procExecutor, 1, true); }
private void cleanupAfterRollbackOneStep(Procedure<TEnvironment> proc) { if (proc.removeStackIndex()) { if (!proc.isSuccess()) { proc.setState(ProcedureState.ROLLEDBACK); } // update metrics on finishing the procedure (fail) proc.updateMetricsOnFinish(getEnvironment(), proc.elapsedTime(), false); if (proc.hasParent()) { store.delete(proc.getProcId()); procedures.remove(proc.getProcId()); } else { final long[] childProcIds = rollbackStack.get(proc.getProcId()).getSubprocedureIds(); if (childProcIds != null) { store.delete(proc, childProcIds); } else { store.update(proc); } } } else { store.update(proc); } }
/** * Add a set of new root-procedure to the executor. * @param procs the new procedures to execute. */ // TODO: Do we need to take nonces here? public void submitProcedures(Procedure<TEnvironment>[] procs) { Preconditions.checkArgument(lastProcId.get() >= 0); if (procs == null || procs.length <= 0) { return; } // Prepare procedure for (int i = 0; i < procs.length; ++i) { prepareProcedure(procs[i]).setProcId(nextProcId()); } // Commit the transaction store.insert(procs); if (LOG.isDebugEnabled()) { LOG.debug("Stored " + Arrays.toString(procs)); } // Add the procedure to the executor for (int i = 0; i < procs.length; ++i) { pushProcedure(procs[i]); } }
/** * Internal method called by the ProcedureExecutor that starts the user-level code releaseLock(). */ final void doReleaseLock(TEnvironment env, ProcedureStore store) { locked = false; // persist that we have released the lock. This must be done before we actually release the // lock. Another procedure may take this lock immediately after we release the lock, and if we // crash before persist the information that we have already released the lock, then when // restarting there will be two procedures which both have the lock and cause problems. if (getState() != ProcedureState.ROLLEDBACK) { // If the state is ROLLEDBACK, it means that we have already deleted the procedure from // procedure store, so do not need to log the release operation any more. store.update(this); } releaseLock(env); }
private void load(final boolean abortOnCorruption) throws IOException { Preconditions.checkArgument(completed.isEmpty(), "completed not empty"); Preconditions.checkArgument(rollbackStack.isEmpty(), "rollback state not empty"); Preconditions.checkArgument(procedures.isEmpty(), "procedure map not empty"); Preconditions.checkArgument(scheduler.size() == 0, "run queue not empty"); store.load(new ProcedureStore.ProcedureLoader() { @Override public void setMaxProcId(long maxProcId) { assert lastProcId.get() < 0 : "expected only one call to setMaxProcId()"; lastProcId.set(maxProcId); } @Override public void load(ProcedureIterator procIter) throws IOException { loadProcedures(procIter, abortOnCorruption); } @Override public void handleCorrupted(ProcedureIterator procIter) throws IOException { int corruptedCount = 0; while (procIter.hasNext()) { Procedure<?> proc = procIter.next(); LOG.error("Corrupt " + proc); corruptedCount++; } if (abortOnCorruption && corruptedCount > 0) { throw new IOException("found " + corruptedCount + " corrupted procedure(s) on replay"); } } }); }
store.recoverLease(); et = System.nanoTime(); LOG.info("Recovered {} lease in {}", store.getClass().getSimpleName(),
LOG.trace("Store add " + procedure + " children " + Arrays.toString(subprocs)); store.insert(procedure, subprocs); } else { if (LOG.isTraceEnabled()) { LOG.trace("Store update " + procedure); store.update(procedure); if (!store.isRunning()) { return; store.update(parent); runnables.addFront(parent); if (LOG.isTraceEnabled()) {
private void startProcedureExecutor(final RSProcedureDispatcher remoteDispatcher) throws IOException { final Configuration conf = getConfiguration(); this.procedureStore = new NoopProcedureStore(); this.procedureStore.registerListener(new ProcedureStoreListener() { @Override public void abortProcess() { abort("The Procedure Store lost the lease", null); } }); this.procedureEnv = new MasterProcedureEnv(this, remoteDispatcher != null ? remoteDispatcher : new RSProcedureDispatcher(this)); this.procedureExecutor = new ProcedureExecutor<>(conf, procedureEnv, procedureStore, procedureEnv.getProcedureScheduler()); final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, Math.max(Runtime.getRuntime().availableProcessors(), MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS)); final boolean abortOnCorruption = conf.getBoolean( MasterProcedureConstants.EXECUTOR_ABORT_ON_CORRUPTION, MasterProcedureConstants.DEFAULT_EXECUTOR_ABORT_ON_CORRUPTION); this.procedureStore.start(numThreads); ProcedureTestingUtility.initAndStartWorkers(procedureExecutor, numThreads, abortOnCorruption); this.procedureEnv.getRemoteDispatcher().start(); }
if (!store.isRunning()) { return;
public ProcedureExecutor(final Configuration conf, final TEnvironment environment, final ProcedureStore store, final ProcedureScheduler scheduler) { this.environment = environment; this.scheduler = scheduler; this.store = store; this.conf = conf; this.checkOwnerSet = conf.getBoolean(CHECK_OWNER_SET_CONF_KEY, DEFAULT_CHECK_OWNER_SET); refreshConfiguration(conf); store.registerListener(new ProcedureStoreListener() { @Override public void forceUpdate(long[] procIds) { Arrays.stream(procIds).forEach(procId -> forceUpdateExecutor.execute(() -> { try { forceUpdateProcedure(procId); } catch (IOException e) { LOG.warn("Failed to force update procedure with pid={}", procId); } })); } }); }
@After public void tearDown() throws IOException { procExecutor.stop(); procStore.stop(false); fs.delete(logDir, true); }
@Before public void setUp() throws IOException { htu = new HBaseCommonTestingUtility(); testDir = htu.getDataTestDir(); fs = testDir.getFileSystem(htu.getConfiguration()); logDir = new Path(testDir, "proc-logs"); procStore = ProcedureTestingUtility.createWalStore(htu.getConfiguration(), logDir); procExecutor = new ProcedureExecutor<>(htu.getConfiguration(), new TestProcEnv(), procStore); procStore.start(PROCEDURE_EXECUTOR_SLOTS); ProcedureTestingUtility.initAndStartWorkers(procExecutor, PROCEDURE_EXECUTOR_SLOTS, true); }
private void cleanupAfterRollbackOneStep(Procedure<TEnvironment> proc) { if (proc.removeStackIndex()) { if (!proc.isSuccess()) { proc.setState(ProcedureState.ROLLEDBACK); } // update metrics on finishing the procedure (fail) proc.updateMetricsOnFinish(getEnvironment(), proc.elapsedTime(), false); if (proc.hasParent()) { store.delete(proc.getProcId()); procedures.remove(proc.getProcId()); } else { final long[] childProcIds = rollbackStack.get(proc.getProcId()).getSubprocedureIds(); if (childProcIds != null) { store.delete(proc, childProcIds); } else { store.update(proc); } } } else { store.update(proc); } }