/** * Parses a directory of WALs building up ProcedureState. * For testing parse and profiling. * @param args Include pointer to directory of WAL files for a store instance to parse & load. */ public static void main(String [] args) throws IOException { Configuration conf = HBaseConfiguration.create(); if (args == null || args.length != 1) { System.out.println("ERROR: Empty arguments list; pass path to MASTERPROCWALS_DIR."); System.out.println("Usage: WALProcedureStore MASTERPROCWALS_DIR"); System.exit(-1); } WALProcedureStore store = new WALProcedureStore(conf, new Path(args[0]), null, new WALProcedureStore.LeaseRecovery() { @Override public void recoverFileLease(FileSystem fs, Path path) throws IOException { // no-op } }); try { store.start(16); ProcedureExecutor<?> pe = new ProcedureExecutor<>(conf, new Object()/*Pass anything*/, store); pe.init(1, true); } finally { store.stop(true); } } }
@Test public void testRollAndRemove() throws IOException { // Insert something in the log Procedure<?> proc1 = new TestSequentialProcedure(); procStore.insert(proc1, null); Procedure<?> proc2 = new TestSequentialProcedure(); procStore.insert(proc2, null); // roll the log, now we have 2 procStore.rollWriterForTesting(); assertEquals(2, procStore.getActiveLogs().size()); // everything will be up to date in the second log // so we can remove the first one procStore.update(proc1); procStore.update(proc2); assertEquals(1, procStore.getActiveLogs().size()); // roll the log, now we have 2 procStore.rollWriterForTesting(); assertEquals(2, procStore.getActiveLogs().size()); // remove everything active // so we can remove all the logs procStore.delete(proc1.getProcId()); procStore.delete(proc2.getProcId()); assertEquals(1, procStore.getActiveLogs().size()); }
Path newLogFile = null; long startPos = -1; newLogFile = getLogFilePath(logId); try { newStream = CommonFSUtils.createForWal(fs, newLogFile, false); closeCurrentLogStream(false); buildHoldingCleanupTracker(); } else if (logs.size() > walCountWarnThreshold) { LOG.warn("procedure WALs count={} above the warning threshold {}. check running procedures" + sendForceUpdateSignal(holdingCleanupTracker.getAllActiveProcIds());
private long syncSlots() throws Throwable { int retry = 0; int logRolled = 0; long totalSynced = 0; do { try { totalSynced = syncSlots(stream, slots, 0, slotIndex); break; } catch (Throwable e) { LOG.warn("unable to sync slots, retry=" + retry); if (++retry >= maxRetriesBeforeRoll) { if (logRolled >= maxSyncFailureRoll && isRunning()) { LOG.error("Sync slots after log roll failed, abort.", e); throw e; } if (!rollWriterWithRetries()) { throw e; } logRolled++; retry = 0; } } } while (isRunning()); return totalSynced; }
private boolean rollWriterWithRetries() { for (int i = 0; i < rollRetries && isRunning(); ++i) { if (i > 0) Threads.sleepWithoutInterrupt(waitBeforeRoll * i); try { if (rollWriter()) { return true; } } catch (IOException e) { LOG.warn("Unable to roll the log, attempt=" + (i + 1), e); } } LOG.error(HBaseMarkers.FATAL, "Unable to roll the log"); return false; }
private void periodicRoll() throws IOException { if (storeTracker.isEmpty()) { LOG.trace("no active procedures"); tryRollWriter(); removeAllLogs(flushLogId - 1, "no active procedures"); } else { if (storeTracker.isAllModified()) { LOG.trace("all the active procedures are in the latest log"); removeAllLogs(flushLogId - 1, "all the active procedures are in the latest log"); } // if the log size has exceeded the roll threshold // or the periodic roll timeout is expired, try to roll the wal. if (totalSynced.get() > rollThreshold || getMillisToNextPeriodicRoll() <= 0) { tryRollWriter(); } removeInactiveLogs(); } }
for (int i = 0; i < procs.length; ++i) { procs[i] = new TestProcedure(i + 1, 0); procStore.insert(procs[i], null); procStore.rollWriterForTesting(); for (int i = 0; i < procs.length; ++i) { procStore.update(procs[i]); procStore.rollWriterForTesting(); procStore.stop(false); procStore = new WALProcedureStore(htu.getConfiguration(), logDir, null, new WALProcedureStore.LeaseRecovery() { private int count = 0; procStore.start(PROCEDURE_STORE_SLOTS); procStore.recoverLease(); procStore.load(loader); assertEquals(procs.length, loader.getMaxProcId()); assertEquals(1, loader.getRunnableCount());
lock.lock(); try { while (isRunning()) { try { periodicRoll(); float rollTsSec = getMillisFromLastRoll() / 1000.0f; LOG.trace(String.format("Waiting for data. flushed=%s (%s/sec)", StringUtils.humanSize(totalSynced.get()), waitCond.await(getMillisToNextPeriodicRoll(), TimeUnit.MILLISECONDS); if (slotIndex == 0) { final float rollSec = getMillisFromLastRoll() / 1000.0f; final float syncedPerSec = totalSyncedToStore / rollSec; if (LOG.isTraceEnabled() && (syncWaitMs > 10 || slotIndex < syncMaxSlot)) { long slotSize = syncSlots(); logs.getLast().addToSize(slotSize); totalSyncedToStore = totalSynced.addAndGet(slotSize); Thread.currentThread().interrupt(); syncException.compareAndSet(null, e); sendAbortProcessSignal(); throw e; } catch (Throwable t) { syncException.compareAndSet(null, t); sendAbortProcessSignal();
Mockito.doReturn(firstMaster.getConfiguration()).when(backupMaster3).getConfiguration(); Mockito.doReturn(true).when(backupMaster3).isActiveMaster(); final WALProcedureStore procStore2 = new WALProcedureStore(firstMaster.getConfiguration(), ((WALProcedureStore)procStore).getWALDir(), null, new MasterProcedureEnv.WALStoreLeaseRecovery(backupMaster3)); procStore2.start(1); procStore2.recoverLease(); procStore2.insert(proc2, null); procStore2.delete(proc2.getProcId()); // delete the procedure so that the WAL is removed later
Mockito.doReturn(firstMaster.getConfiguration()).when(backupMaster3).getConfiguration(); Mockito.doReturn(true).when(backupMaster3).isActiveMaster(); final WALProcedureStore backupStore3 = new WALProcedureStore(firstMaster.getConfiguration(), ((WALProcedureStore)masterStore).getWALDir(), null, new MasterProcedureEnv.WALStoreLeaseRecovery(backupMaster3)); backupStore3.registerListener(new ProcedureStore.ProcedureStoreListener() { @Override public void postSync() {} backupStore3.start(1); backupStore3.recoverLease(); backupStore3.getStoreTracker().setDeleted(1, false); try { backupStore3.delete(1); fail("expected RuntimeException 'sync aborted'"); } catch (RuntimeException e) {
private boolean rollWriter() throws IOException { if (!isRunning()) { return false; } // Create new state-log if (!rollWriter(flushLogId + 1)) { LOG.warn("someone else has already created log {}", flushLogId); return false; } // We have the lease on the log, // but we should check if someone else has created new files if (getMaxLogId(getLogFiles()) > flushLogId) { LOG.warn("Someone else created new logs. Expected maxLogId < {}", flushLogId); logs.getLast().removeFile(this.walArchiveDir); return false; } // We have the lease on the log return true; }
procStore.insert(proc0, null); Procedure<?> proc1 = new TestSequentialProcedure(); procStore.insert(proc1, null); Procedure<?> proc2 = new TestSequentialProcedure(); procStore.insert(proc2, null); procStore.rollWriterForTesting(); procStore.delete(proc1.getProcId()); procStore.rollWriterForTesting(); procStore.update(proc2); procStore.rollWriterForTesting(); procStore.delete(proc2.getProcId()); procStore.stop(false); FileStatus[] logs = fs.listStatus(logDir); assertEquals(4, logs.length); procStore.delete(proc0.getProcId()); procStore.periodicRollForTesting(); assertEquals(1, fs.listStatus(logDir).length); storeRestart(loader);
@Test public void testWalCleanerUpdates() throws Exception { TestSequentialProcedure p1 = new TestSequentialProcedure(); TestSequentialProcedure p2 = new TestSequentialProcedure(); procStore.insert(p1, null); procStore.insert(p2, null); procStore.rollWriterForTesting(); ProcedureWALFile firstLog = procStore.getActiveLogs().get(0); procStore.update(p1); procStore.rollWriterForTesting(); procStore.update(p2); procStore.rollWriterForTesting(); procStore.removeInactiveLogsForTesting(); assertFalse(procStore.getActiveLogs().contains(firstLog)); }
private void createProcedureExecutor() throws IOException { MasterProcedureEnv procEnv = new MasterProcedureEnv(this); procedureStore = new WALProcedureStore(conf, new MasterProcedureEnv.WALStoreLeaseRecovery(this)); procedureStore.registerListener(new ProcedureStoreListener() { @Override public void abortProcess() { abort("The Procedure Store lost the lease", null); } }); MasterProcedureScheduler procedureScheduler = procEnv.getProcedureScheduler(); procedureExecutor = new ProcedureExecutor<>(conf, procEnv, procedureStore, procedureScheduler); configurationManager.registerObserver(procEnv); int cpus = Runtime.getRuntime().availableProcessors(); final int numThreads = conf.getInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, Math.max( (cpus > 0 ? cpus / 4 : 0), MasterProcedureConstants.DEFAULT_MIN_MASTER_PROCEDURE_THREADS)); final boolean abortOnCorruption = conf.getBoolean(MasterProcedureConstants.EXECUTOR_ABORT_ON_CORRUPTION, MasterProcedureConstants.DEFAULT_EXECUTOR_ABORT_ON_CORRUPTION); procedureStore.start(numThreads); // Just initialize it but do not start the workers, we will start the workers later by calling // startProcedureExecutor. See the javadoc for finishActiveMasterInitialization for more // details. procedureExecutor.init(numThreads, abortOnCorruption); procEnv.getRemoteDispatcher().start(); }
public void setUpProcedureStore() throws IOException { Path testDir = UTIL.getDataTestDir(); FileSystem fs = testDir.getFileSystem(conf); Path logDir = new Path(testDir, "proc-logs"); System.out.println("\n\nLogs directory : " + logDir.toString() + "\n\n"); fs.delete(logDir, true); store = ProcedureTestingUtility.createWalStore(conf, logDir); store.start(1); store.recoverLease(); store.load(new LoadCounter()); }
private void storeRestart(ProcedureStore.ProcedureLoader loader) throws IOException { System.out.println("Restarting procedure store to read back the WALs"); store.stop(false); store.start(1); store.recoverLease(); long startTime = currentTimeMillis(); store.load(loader); long timeTaken = System.currentTimeMillis() - startTime; System.out.println("******************************************"); System.out.println("Load time : " + (timeTaken / 1000.0f) + "sec"); System.out.println("******************************************"); System.out.println("Raw format for scripts"); System.out.println(String.format("RESULT [%s=%s, %s=%s, %s=%s, %s=%s, %s=%s, " + "total_time_ms=%s]", NUM_PROCS_OPTION.getOpt(), numProcs, STATE_SIZE_OPTION.getOpt(), serializedState.length, UPDATES_PER_PROC_OPTION.getOpt(), updatesPerProc, DELETE_PROCS_FRACTION_OPTION.getOpt(), deleteProcsFraction, NUM_WALS_OPTION.getOpt(), numWals, timeTaken)); }
private void setupProcedureStore() throws IOException { Path testDir = UTIL.getDataTestDir(); FileSystem fs = testDir.getFileSystem(conf); Path logDir = new Path(testDir, "proc-logs"); System.out.println("Logs directory : " + logDir.toString()); fs.delete(logDir, true); if ("nosync".equals(syncType)) { store = new NoSyncWalProcedureStore(conf, logDir); } else { store = ProcedureTestingUtility.createWalStore(conf, logDir); } store.start(numThreads); store.recoverLease(); store.load(new ProcedureTestingUtility.LoadCounter()); System.out.println("Starting new log : " + store.getActiveLogs().get(store.getActiveLogs().size() - 1)); }
WALProcedureStore walStore = master.getWalProcedureStore(); ArrayList<WALProcedureStore.SyncMetrics> syncMetricsBuff = walStore.getSyncMetrics(); long millisToNextRoll = walStore.getMillisToNextPeriodicRoll(); long millisFromLastRoll = walStore.getMillisFromLastRoll(); ArrayList<ProcedureWALFile> procedureWALFiles = walStore.getActiveLogs(); Set<ProcedureWALFile> corruptedWALFiles = walStore.getCorruptedLogs(); List<Procedure<MasterProcedureEnv>> procedures = procExecutor.getProcedures(); Collections.sort(procedures, new Comparator<Procedure>() {
@Override public void abortProcess() { LOG.error(HBaseMarkers.FATAL, "Abort the Procedure Store"); store.stop(true); } };
procStore.insert(proc1, null); procIds.add(child2[0].getProcId()); procIds.add(child2[1].getProcId()); procStore.insert(proc2, child2); procStore.update(proc1); procStore.update(child2[1]); procStore.delete(child2[1].getProcId()); procIds.remove(child2[1].getProcId()); procStore.stop(false); FileStatus[] logs = fs.listStatus(logDir); assertEquals(3, logs.length);