private void createProcedureExecutor() throws IOException { MasterProcedureEnv procEnv = new MasterProcedureEnv(this); procedureStore = new WALProcedureStore(conf, new MasterProcedureEnv.WALStoreLeaseRecovery(this)); procedureStore.registerListener(new ProcedureStoreListener() { @Override public void abortProcess() { abort("The Procedure Store lost the lease", null); } }); MasterProcedureScheduler procedureScheduler = procEnv.getProcedureScheduler(); procedureExecutor = new ProcedureExecutor<>(conf, procEnv, procedureStore, procedureScheduler); configurationManager.registerObserver(procEnv); int cpus = Runtime.getRuntime().availableProcessors(); final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, Math.max( (cpus > 0 ? cpus / 4 : 0), MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS)); final boolean abortOnCorruption = conf.getBoolean(MasterProcedureConstants.EXECUTOR_ABORT_ON_CORRUPTION, MasterProcedureConstants.DEFAULT_EXECUTOR_ABORT_ON_CORRUPTION); procedureStore.start(numThreads); // Just initialize it but do not start the workers, we will start the workers later by calling // startProcedureExecutor. See the javadoc for finishActiveMasterInitialization for more // details. procedureExecutor.init(numThreads, abortOnCorruption); procEnv.getRemoteDispatcher().start(); }
private void setupDFS() throws Exception { Configuration conf = UTIL.getConfiguration(); MiniDFSCluster dfs = UTIL.startMiniDFSCluster(3); CommonFSUtils.setWALRootDir(conf, new Path(conf.get("fs.defaultFS"), "/tmp/wal")); Path logDir = new Path(new Path(dfs.getFileSystem().getUri()), "/test-logs"); store = ProcedureTestingUtility.createWalStore(conf, logDir); store.registerListener(stopProcedureListener); store.start(8); store.recoverLease(); }
store.registerListener(new ProcedureStore.ProcedureStoreListener() { @Override public void postSync() { Threads.sleepWithoutInterrupt(2000); }
backupStore3.registerListener(new ProcedureStore.ProcedureStoreListener() { @Override public void postSync() {}
private void startProcedureExecutor() throws IOException { final MasterProcedureEnv procEnv = new MasterProcedureEnv(this); final Path logDir = new Path(fileSystemManager.getRootDir(), MasterProcedureConstants.MASTER_PROCEDURE_LOGDIR); procedureStore = new WALProcedureStore(conf, fileSystemManager.getFileSystem(), logDir, new MasterProcedureEnv.WALStoreLeaseRecovery(this)); procedureStore.registerListener(new MasterProcedureEnv.MasterProcedureStoreListener(this)); procedureExecutor = new ProcedureExecutor(conf, procEnv, procedureStore, procEnv.getProcedureQueue()); final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, Math.max(Runtime.getRuntime().availableProcessors(), MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS)); final boolean abortOnCorruption = conf.getBoolean( MasterProcedureConstants.EXECUTOR_ABORT_ON_CORRUPTION, MasterProcedureConstants.DEFAULT_EXECUTOR_ABORT_ON_CORRUPTION); procedureStore.start(numThreads); procedureExecutor.start(numThreads, abortOnCorruption); }
private void setupDFS() throws Exception { MiniDFSCluster dfs = UTIL.startMiniDFSCluster(3); Path logDir = new Path(new Path(dfs.getFileSystem().getUri()), "/test-logs"); store = ProcedureTestingUtility.createWalStore(UTIL.getConfiguration(), logDir); store.registerListener(stopProcedureListener); store.start(8); store.recoverLease(); }
store.registerListener(new ProcedureStore.ProcedureStoreListener() { @Override public void postSync() { Threads.sleepWithoutInterrupt(2000); }
backupStore3.registerListener(new ProcedureStore.ProcedureStoreListener() { @Override public void postSync() {}