/** * pad the current file to increase its size to the next multiple of preAllocSize greater than the current size and position * * @param fileChannel the fileChannel of the file to be padded * @throws IOException */ long padFile(FileChannel fileChannel) throws IOException { long newFileSize = calculateFileSizeWithPadding(fileChannel.position(), currentSize, preAllocSize); if (currentSize != newFileSize) { fileChannel.write((ByteBuffer) fill.position(0), newFileSize - fill.remaining()); currentSize = newFileSize; } return currentSize; }
/** * method to allow setting preallocate size * of log file to pad the file. * @param size the size to set to in bytes */ public static void setPreallocSize(long size) { FilePadding.setPreallocSize(size); }
filePadding.setCurrentSize(fos.getChannel().position()); streamsToFlush.add(fos); filePadding.padFile(fos.getChannel()); byte[] buf = Util.marshallTxnEntry(hdr, txn); if (buf == null || buf.length == 0) {
@Test public void testSetPreallocSize() { long customPreallocSize = 10101; FileTxnLog.setPreallocSize(customPreallocSize); Assert.assertThat(FilePadding.getPreAllocSize(), is(equalTo(customPreallocSize))); }
fhdr.serialize(recoveryOa, "fileheader"); recoveryFos.flush(); filePadding.setCurrentSize(recoveryFos.getChannel().position()); filePadding.padFile(recoveryFos.getChannel()); recoveryOa.writeLong(crcValue, "crcvalue"); recoveryOa.writeBuffer(bytes, "txnEntry");
filePadding.setCurrentSize(fos.getChannel().position()); streamsToFlush.add(fos); filePadding.padFile(fos.getChannel()); byte[] buf = Util.marshallTxnEntry(hdr, txn); if (buf == null || buf.length == 0) {
/** * pad the current file to increase its size to the next multiple of preAllocSize greater than the current size and position * * @param fileChannel the fileChannel of the file to be padded * @throws IOException */ long padFile(FileChannel fileChannel) throws IOException { long newFileSize = calculateFileSizeWithPadding(fileChannel.position(), currentSize, preAllocSize); if (currentSize != newFileSize) { fileChannel.write((ByteBuffer) fill.position(0), newFileSize - fill.remaining()); currentSize = newFileSize; } return currentSize; }
/** * method to allow setting preallocate size * of log file to pad the file. * @param size the size to set to in bytes */ public static void setPreallocSize(long size) { FilePadding.setPreallocSize(size); }
fhdr.serialize(recoveryOa, "fileheader"); recoveryFos.flush(); filePadding.setCurrentSize(recoveryFos.getChannel().position()); filePadding.padFile(recoveryFos.getChannel()); recoveryOa.writeLong(crcValue, "crcvalue"); recoveryOa.writeBuffer(bytes, "txnEntry");
@Test public void testInvalidPreallocSize() { Assert.assertEquals("file should not be padded", 10 * KB, FilePadding.calculateFileSizeWithPadding(7 * KB, 10 * KB, 0)); Assert.assertEquals("file should not be padded", 10 * KB, FilePadding.calculateFileSizeWithPadding(7 * KB, 10 * KB, -1)); }
/** * Test specific setup */ public static void setupTestEnv() { // during the tests we run with 100K prealloc in the logs. // on windows systems prealloc of 64M was seen to take ~15seconds // resulting in test Assert.failure (client timeout on first session). // set env and directly in order to handle static init/gc issues System.setProperty("zookeeper.preAllocSize", "100"); FilePadding.setPreallocSize(100 * 1024); }
@Test public void testCalculateFileSizeWithPaddingWhenNotToCurrentSize() { Assert.assertEquals("file should not be padded", 10 * KB, FilePadding.calculateFileSizeWithPadding(5 * KB, 10 * KB, 10 * KB)); }
@Test public void testPreAllocSizeSmallerThanTxnData() throws IOException { File logDir = ClientBase.createTmpDir(); FileTxnLog fileTxnLog = new FileTxnLog(logDir); // Set a small preAllocSize (.5 MB) final int preAllocSize = 500 * KB; FilePadding.setPreallocSize(preAllocSize); // Create dummy txn larger than preAllocSize // Since the file padding inserts a 0, we will fill the data with 0xff to ensure we corrupt the data if we put the 0 in the data byte[] data = new byte[2 * preAllocSize]; Arrays.fill(data, (byte) 0xff); // Append and commit 2 transactions to the log // Prior to ZOOKEEPER-2249, attempting to pad in association with the second transaction will corrupt the first fileTxnLog.append(new TxnHeader(1, 1, 1, 1, ZooDefs.OpCode.create), new CreateTxn("/testPreAllocSizeSmallerThanTxnData1", data, ZooDefs.Ids.OPEN_ACL_UNSAFE, false, 0)); fileTxnLog.commit(); fileTxnLog.append(new TxnHeader(1, 1, 2, 2, ZooDefs.OpCode.create), new CreateTxn("/testPreAllocSizeSmallerThanTxnData2", new byte[]{}, ZooDefs.Ids.OPEN_ACL_UNSAFE, false, 0)); fileTxnLog.commit(); fileTxnLog.close(); // Read the log back from disk, this will throw a java.io.IOException: CRC check failed prior to ZOOKEEPER-2249 FileTxnLog.FileTxnIterator fileTxnIterator = new FileTxnLog.FileTxnIterator(logDir, 0); // Verify the data in the first transaction CreateTxn createTxn = (CreateTxn) fileTxnIterator.getTxn(); Assert.assertTrue(Arrays.equals(createTxn.getData(), data)); // Verify the data in the second transaction fileTxnIterator.next(); createTxn = (CreateTxn) fileTxnIterator.getTxn(); Assert.assertTrue(Arrays.equals(createTxn.getData(), new byte[]{})); }
@Test public void testCalculateFileSizeWithPaddingWhenCloseToCurrentSize() { Assert.assertEquals("file should be padded an additional 10 KB", 20 * KB, FilePadding.calculateFileSizeWithPadding(7 * KB, 10 * KB, 10 * KB)); }
@Test public void testFileSizeGreaterThanPosition() { Assert.assertEquals("file should be padded to 40 KB", 40 * KB, FilePadding.calculateFileSizeWithPadding(31 * KB, 10 * KB, 10 * KB)); }