private static void setupTestEnv() { // during the tests we run with 100K prealloc in the logs. // on windows systems prealloc of 64M was seen to take ~15seconds // resulting in test failure (client timeout on first session). // set env and directly in order to handle static init/gc issues System.setProperty("zookeeper.preAllocSize", "100"); FileTxnLog.setPreallocSize(100 * 1024); // allow all 4 letter words System.setProperty("zookeeper.4lw.commands.whitelist","*"); }
/** * Test that log size get update correctly */ @Test public void testGetCurrentLogSize() throws Exception { FileTxnLog.setTxnLogSizeLimit(-1); File tmpDir = ClientBase.createTmpDir(); FileTxnLog log = new FileTxnLog(tmpDir); FileTxnLog.setPreallocSize(PREALLOCATE); CreateRequest record = new CreateRequest(null, new byte[NODE_SIZE], ZooDefs.Ids.OPEN_ACL_UNSAFE, 0); int zxid = 1; for (int i = 0; i < 4; i++) { log.append(new TxnHeader(0, 0, zxid++, 0, 0), record); LOG.debug("Current log size: " + log.getCurrentLogSize()); } log.commit(); LOG.info("Current log size: " + log.getCurrentLogSize()); Assert.assertTrue(log.getCurrentLogSize() > (zxid - 1) * NODE_SIZE); for (int i = 0; i < 4; i++) { log.append(new TxnHeader(0, 0, zxid++, 0, 0), record); LOG.debug("Current log size: " + log.getCurrentLogSize()); } log.commit(); LOG.info("Current log size: " + log.getCurrentLogSize()); Assert.assertTrue(log.getCurrentLogSize() > (zxid - 1) * NODE_SIZE); }
/** * get the last zxid that was logged in the transaction logs * @return the last zxid logged in the transaction logs */ public long getLastLoggedZxid() { File[] files = getLogFiles(logDir.listFiles(), 0); long maxLog=files.length>0? Util.getZxidFromName(files[files.length-1].getName(),LOG_FILE_PREFIX):-1; // if a log file is more recent we must scan it to find // the highest zxid long zxid = maxLog; TxnIterator itr = null; try { FileTxnLog txn = new FileTxnLog(logDir); itr = txn.read(maxLog); while (true) { if(!itr.next()) break; TxnHeader hdr = itr.getHeader(); zxid = hdr.getZxid(); } } catch (IOException e) { LOG.warn("Unexpected exception", e); } finally { close(itr); } return zxid; }
/** * the last logged zxid on the transaction logs * @return the last logged zxid */ public long getLastLoggedZxid() { FileTxnLog txnLog = new FileTxnLog(dataDir); return txnLog.getLastLoggedZxid(); }
/** * Get TxnIterator for iterating through txnlog starting at a given zxid * * @param zxid starting zxid * @param fastForward true if the iterator should be fast forwarded to point * to the txn of a given zxid, else the iterator will point to the * starting txn of a txnlog that may contain txn of a given zxid * @return TxnIterator * @throws IOException */ public TxnIterator readTxnLog(long zxid, boolean fastForward) throws IOException { FileTxnLog txnLog = new FileTxnLog(dataDir); return txnLog.read(zxid, fastForward); }
/** * truncate the transaction logs the zxid * specified * @param zxid the zxid to truncate the logs to * @return true if able to truncate the log, false if not * @throws IOException */ public boolean truncateLog(long zxid) throws IOException { // close the existing txnLog and snapLog close(); // truncate it FileTxnLog truncLog = new FileTxnLog(dataDir); boolean truncated = truncLog.truncate(zxid); truncLog.close(); // re-open the txnLog and snapLog // I'd rather just close/reopen this object itself, however that // would have a big impact outside ZKDatabase as there are other // objects holding a reference to this object. txnLog = new FileTxnLog(dataDir); snapLog = new FileSnap(snapDir); return truncated; }
FileTxnLog.setPreallocSize(PREALLOCATE); FileTxnLog.setTxnLogSizeLimit(LOG_SIZE_LIMIT); File[] txnLogs = FileTxnLog.getLogFiles(logDir.listFiles(), 0);
File[] logFiles = FileTxnLog.getLogFiles(logDir.listFiles(), 0); FileTxnLog txnLog = new FileTxnLog(logDir); TxnIterator itr = txnLog.read(1, false); itr = txnLog.read(secondStartZxid, false); Assert.assertEquals(secondStartZxid, itr.getHeader().getZxid()); Assert.assertTrue(itr.next()); itr = txnLog.read(nextZxid, false); Assert.assertEquals(secondStartZxid, itr.getHeader().getZxid()); itr = txnLog.read(thirdStartZxid, false); Assert.assertEquals(secondStartZxid, itr.getHeader().getZxid()); Assert.assertTrue(itr.next()); itr = txnLog.read(nextZxid, false); Assert.assertEquals(secondStartZxid, itr.getHeader().getZxid());
/** * get the snapshot logs which may contain transactions newer than the given zxid. * This includes logs with starting zxid greater than given zxid, as well as the * newest transaction log with starting zxid less than given zxid. The latter log * file may contain transactions beyond given zxid. * @param zxid the zxid that contains logs greater than * zxid * @return */ public File[] getSnapshotLogs(long zxid) { return FileTxnLog.getLogFiles(dataDir.listFiles(), zxid); }
Pair<Long, Long> getFirstLastZxid(File logFile) throws IOException { File tmp = createTmpDir(); Files.copy(logFile.toPath(), new File(tmp, "log.0").toPath()); FileTxnLog txnLog = new FileTxnLog(tmp); TxnLog.TxnIterator it = txnLog.read(0); long firstZxid = it.getHeader().getZxid(); long lastZxid = firstZxid; while (it.next()) { lastZxid = it.getHeader().getZxid(); } txnLog.close(); rmr(tmp); return new Pair<Long, Long>(firstZxid, lastZxid); }
@Test public void testPreAllocSizeSmallerThanTxnData() throws IOException { File logDir = ClientBase.createTmpDir(); FileTxnLog fileTxnLog = new FileTxnLog(logDir); // Set a small preAllocSize (.5 MB) final int preAllocSize = 500 * KB; FilePadding.setPreallocSize(preAllocSize); // Create dummy txn larger than preAllocSize // Since the file padding inserts a 0, we will fill the data with 0xff to ensure we corrupt the data if we put the 0 in the data byte[] data = new byte[2 * preAllocSize]; Arrays.fill(data, (byte) 0xff); // Append and commit 2 transactions to the log // Prior to ZOOKEEPER-2249, attempting to pad in association with the second transaction will corrupt the first fileTxnLog.append(new TxnHeader(1, 1, 1, 1, ZooDefs.OpCode.create), new CreateTxn("/testPreAllocSizeSmallerThanTxnData1", data, ZooDefs.Ids.OPEN_ACL_UNSAFE, false, 0)); fileTxnLog.commit(); fileTxnLog.append(new TxnHeader(1, 1, 2, 2, ZooDefs.OpCode.create), new CreateTxn("/testPreAllocSizeSmallerThanTxnData2", new byte[]{}, ZooDefs.Ids.OPEN_ACL_UNSAFE, false, 0)); fileTxnLog.commit(); fileTxnLog.close(); // Read the log back from disk, this will throw a java.io.IOException: CRC check failed prior to ZOOKEEPER-2249 FileTxnLog.FileTxnIterator fileTxnIterator = new FileTxnLog.FileTxnIterator(logDir, 0); // Verify the data in the first transaction CreateTxn createTxn = (CreateTxn) fileTxnIterator.getTxn(); Assert.assertTrue(Arrays.equals(createTxn.getData(), data)); // Verify the data in the second transaction fileTxnIterator.next(); createTxn = (CreateTxn) fileTxnIterator.getTxn(); Assert.assertTrue(Arrays.equals(createTxn.getData(), new byte[]{})); }
public void testSyncThresholdExceedCount() throws IOException { // Given ... // Set threshold to -1, as after the first commit it takes 0ms to commit to disk. java.lang.System.setProperty(FileTxnLog.ZOOKEEPER_FSYNC_WARNING_THRESHOLD_MS_PROPERTY, "-1"); ServerStats.Provider providerMock = mock(ServerStats.Provider.class); ServerStats serverStats = new ServerStats(providerMock); File logDir = ClientBase.createTmpDir(); FileTxnLog fileTxnLog = new FileTxnLog(logDir); fileTxnLog.setServerStats(serverStats); // Verify serverStats is 0 before any commit Assert.assertEquals(0L, serverStats.getFsyncThresholdExceedCount()); // When ... for (int i = 0; i < 50; i++) { fileTxnLog.append(new TxnHeader(1, 1, 1, 1, ZooDefs.OpCode.create), new CreateTxn("/testFsyncThresholdCountIncreased", new byte[]{}, ZooDefs.Ids.OPEN_ACL_UNSAFE, false, 0)); fileTxnLog.commit(); // only 1 commit, otherwise it will be flaky // Then ... verify serverStats is updated to the number of commits (as threshold is set to 0) Assert.assertEquals((long) i + 1 , serverStats.getFsyncThresholdExceedCount()); } }
@Test public void testForceSyncDefaultEnabled() { File file = new File("foo.10027c6de"); FileTxnLog log = new FileTxnLog(file); Assert.assertTrue(log.isForceSync()); }
/** * truncate the transaction logs the zxid * specified * @param zxid the zxid to truncate the logs to * @return true if able to truncate the log, false if not * @throws IOException */ public boolean truncateLog(long zxid) throws IOException { FileTxnLog txnLog = new FileTxnLog(dataDir); return txnLog.truncate(zxid); }
txnLog = new FileTxnLog(this.dataDir); snapLog = new FileSnap(this.snapDir);
DeleteTxn txn = new DeleteTxn("/foo"); File tmpDir = createTmpDir(); FileTxnLog txnLog = new FileTxnLog(tmpDir); txnLog.append(hdr, txn); txnLog.append(hdr, txn); txnLog.commit();
/** * start reading all the transactions from the given zxid * @param zxid the zxid to start reading transactions from * @return returns an iterator to iterate through the transaction * logs */ public TxnIterator read(long zxid) throws IOException { return read(zxid, true); }
/** * Simulates ZOOKEEPER-1069 and verifies that flush() before padLogFile * fixes it. */ @Test public void testPad() throws Exception { File tmpDir = ClientBase.createTmpDir(); FileTxnLog txnLog = new FileTxnLog(tmpDir); TxnHeader txnHeader = new TxnHeader(0xabcd, 0x123, 0x123, Time.currentElapsedTime(), ZooDefs.OpCode.create); Record txn = new CreateTxn("/Test", new byte[0], null, false, 1); txnLog.append(txnHeader, txn); FileInputStream in = new FileInputStream(tmpDir.getPath() + "/log." + Long.toHexString(txnHeader.getZxid())); BinaryInputArchive ia = BinaryInputArchive.getArchive(in); FileHeader header = new FileHeader(); header.deserialize(ia, "fileheader"); LOG.info("Received magic : " + header.getMagic() + " Expected : " + FileTxnLog.TXNLOG_MAGIC); Assert.assertTrue("Missing magic number ", header.getMagic() == FileTxnLog.TXNLOG_MAGIC); }
/** * truncate the current transaction logs * @param zxid the zxid to truncate the logs to * @return true if successful false if not */ public boolean truncate(long zxid) throws IOException { FileTxnIterator itr = null; try { itr = new FileTxnIterator(this.logDir, zxid); PositionInputStream input = itr.inputStream; if(input == null) { throw new IOException("No log files found to truncate! This could " + "happen if you still have snapshots from an old setup or " + "log files were deleted accidentally or dataLogDir was changed in zoo.cfg."); } long pos = input.getPosition(); // now, truncate at the current position RandomAccessFile raf=new RandomAccessFile(itr.logFile,"rw"); raf.setLength(pos); raf.close(); while(itr.goToNextLog()) { if (!itr.logFile.delete()) { LOG.warn("Unable to truncate {}", itr.logFile); } } } finally { close(itr); } return true; }
FileTxnLog txnLog = new FileTxnLog(dataDir); TxnIterator itr = txnLog.read(dt.lastProcessedZxid+1); long highestZxid = dt.lastProcessedZxid; TxnHeader hdr;