@Test public void testCompactFirstFileReclaimed() throws Exception { setup(2, 60 * 1024, false); final byte recordType = (byte) 0; journal = new JournalImpl(fileSize, minFiles, minFiles, 0, 0, fileFactory, filePrefix, fileExtension, maxAIO); journal.start(); journal.loadInternalOnly(); journal.appendAddRecord(1, recordType, "test".getBytes(), true); journal.forceMoveNextFile(); journal.appendUpdateRecord(1, recordType, "update".getBytes(), true); journal.appendDeleteRecord(1, true); journal.appendAddRecord(2, recordType, "finalRecord".getBytes(), true); for (int i = 10; i < 100; i++) { journal.appendAddRecord(i, recordType, ("tst" + i).getBytes(), true); journal.forceMoveNextFile(); journal.appendUpdateRecord(i, recordType, ("uptst" + i).getBytes(), true); journal.appendDeleteRecord(i, true); } journal.testCompact(); journal.stop(); List<RecordInfo> records1 = new ArrayList<>(); List<PreparedTransactionInfo> preparedRecords = new ArrayList<>(); journal.start(); journal.load(records1, preparedRecords, null); assertEquals(1, records1.size()); }
@Test public void testOrganicallyWithALimit() throws Exception { setup(2, 5, 10 * 1024, true, 50); createJournal(); journal.setAutoReclaim(true); startJournal(); load(); List<String> files1 = fileFactory.listFiles(fileExtension); Assert.assertEquals(2, files1.size()); Assert.assertEquals(0, journal.getDataFilesCount()); Assert.assertEquals(0, journal.getFreeFilesCount()); Assert.assertEquals(1, journal.getOpenedFilesCount()); Assert.assertEquals(0, journal.getIDMapSize()); // Fill all the files for (int i = 0; i < 200; i++) { add(i); journal.forceMoveNextFile(); } journal.checkReclaimStatus(); for (int i = 0; i < 200; i++) { delete(i); } journal.forceMoveNextFile(); journal.checkReclaimStatus(); files1 = fileFactory.listFiles(fileExtension); Assert.assertTrue("supposed to have less than 10 but it had " + files1.size() + " files created", files1.size() < 10); stopJournal(); }
protected void checkAndReclaimFiles() throws Exception { journal.debugWait(); boolean originalAutoReclaim = journal.isAutoReclaim(); journal.setAutoReclaim(true); journal.checkReclaimStatus(); journal.setAutoReclaim(originalAutoReclaim); journal.debugWait(); }
protected void delete(final long... arguments) throws Exception { for (long element : arguments) { beforeJournalOperation(); journal.appendDeleteRecord(element, sync); removeRecordsForID(element); } journal.debugWait(); }
@Test public void testAddDeleteCommitTXIDMap1() throws Exception { setup(10, 10 * 1024, true); createJournal(); startJournal(); load(); List<String> files1 = fileFactory.listFiles(fileExtension); Assert.assertEquals(10, files1.size()); Assert.assertEquals(0, journal.getDataFilesCount()); Assert.assertEquals(8, journal.getFreeFilesCount()); Assert.assertEquals(1, journal.getOpenedFilesCount()); Assert.assertEquals(0, journal.getIDMapSize()); addTx(1, 1); Assert.assertEquals(0, journal.getDataFilesCount()); Assert.assertEquals(8, journal.getFreeFilesCount()); Assert.assertEquals(1, journal.getOpenedFilesCount()); Assert.assertEquals(0, journal.getIDMapSize()); deleteTx(1, 1); Assert.assertEquals(0, journal.getDataFilesCount()); Assert.assertEquals(8, journal.getFreeFilesCount()); Assert.assertEquals(1, journal.getOpenedFilesCount()); Assert.assertEquals(0, journal.getIDMapSize()); commit(1); Assert.assertEquals(0, journal.getDataFilesCount()); Assert.assertEquals(8, journal.getFreeFilesCount()); Assert.assertEquals(1, journal.getOpenedFilesCount()); Assert.assertEquals(0, journal.getIDMapSize()); }
journal.forceMoveNextFile(); journal.forceMoveNextFile(); journal.forceMoveNextFile(); journal.forceMoveNextFile(); journal.checkReclaimStatus(); journal.testCompact();
Assert.assertEquals(0, journal.getDataFilesCount()); Assert.assertEquals(0, journal.getFreeFilesCount()); Assert.assertEquals(1, journal.getOpenedFilesCount()); Assert.assertEquals(0, journal.getIDMapSize()); journal.debugWait(); System.out.println("journal tmp :" + journal.debug()); Assert.assertEquals(1, journal.getOpenedFilesCount()); Assert.assertEquals(1, journal.getDataFilesCount()); Assert.assertEquals(0, journal.getFreeFilesCount()); Assert.assertEquals(0, journal.getIDMapSize()); journal.debugWait(); System.out.println("journal tmp2 :" + journal.debug()); Assert.assertEquals(1, journal.getOpenedFilesCount()); UnitTestLogger.LOGGER.debug("data files count " + journal.getDataFilesCount()); UnitTestLogger.LOGGER.debug("free files count " + journal.getFreeFilesCount()); Assert.assertEquals(2, journal.getDataFilesCount()); Assert.assertEquals(0, journal.getFreeFilesCount()); Assert.assertEquals(1, journal.getIDMapSize()); Assert.assertEquals(1, journal.getOpenedFilesCount());
add(id); if (i > 0 && i % 100 == 0) { journal.forceMoveNextFile(); journal.forceMoveNextFile(); journal.checkReclaimStatus(); add(i); if (i % 10 == 0 && i > 0) { journal.forceMoveNextFile(); updateTx(transactionID, i); if (i % 10 == 0) { journal.forceMoveNextFile(); journal.forceMoveNextFile(); System.out.println("Number of Files: " + journal.getDataFilesCount()); System.out.println(journal.debug()); System.out.println("*****************************************"); journal.testCompact(); journal.testCompact();
Assert.assertEquals(0, journal.getDataFilesCount()); Assert.assertEquals(0, journal.getFreeFilesCount()); Assert.assertEquals(1, journal.getOpenedFilesCount()); Assert.assertEquals(0, journal.getIDMapSize()); Assert.assertEquals(0, journal.getDataFilesCount()); Assert.assertEquals(0, journal.getFreeFilesCount()); Assert.assertEquals(1, journal.getOpenedFilesCount()); Assert.assertEquals(0, journal.getIDMapSize()); journal.forceMoveNextFile(); journal.debugWait(); Assert.assertEquals(1, journal.getDataFilesCount()); Assert.assertEquals(0, journal.getFreeFilesCount()); Assert.assertEquals(1, journal.getIDMapSize()); Assert.assertEquals(1, journal.getDataFilesCount()); Assert.assertEquals(0, journal.getFreeFilesCount()); Assert.assertEquals(1, journal.getOpenedFilesCount()); Assert.assertEquals(1, journal.getIDMapSize()); Assert.assertEquals(1, journal.getOpenedFilesCount()); Assert.assertEquals(1, journal.getDataFilesCount()); Assert.assertEquals(0, journal.getFreeFilesCount()); Assert.assertEquals(0, journal.getIDMapSize());
@Test public void testCompactAddAndUpdateFollowedByADelete2() throws Exception { setup(2, 60 * 1024, false); SimpleIDGenerator idGen = new SimpleIDGenerator(1000); createJournal(); journal.setAutoReclaim(false); startJournal(); load(); long firstID = idGen.generateID(); long consumerTX = idGen.generateID(); long appendTX = idGen.generateID(); long addedRecord = idGen.generateID(); addTx(consumerTX, firstID); startCompact(); addTx(appendTX, addedRecord); commit(appendTX); updateTx(consumerTX, addedRecord); commit(consumerTX); long deleteTXID = idGen.generateID(); deleteTx(deleteTXID, addedRecord); commit(deleteTXID); finishCompact(); journal.forceMoveNextFile(); journal.testCompact(); stopJournal(); createJournal(); startJournal(); loadAndCheck(); }
journal.setAutoReclaim(false); journal.forceMoveNextFile(); update(newRecord); journal.testCompact(); System.out.println("Debug after compact\n" + journal.debug());
journal.setAutoReclaim(false); add(i); if (i % 10 == 0 && i > 0) { journal.forceMoveNextFile(); updateTx(transactionID, i); if (i % 10 == 0) { journal.forceMoveNextFile(); journal.forceMoveNextFile(); add(nextID++); if (i % 10 == 0) { journal.forceMoveNextFile(); commit(transactionID++); if (i % 10 == 0) { journal.forceMoveNextFile(); journal.forceMoveNextFile(); journal.forceMoveNextFile(); journal.testCompact(); journal.flush(); journal.forceMoveNextFile();
@Test public void testSimpleCommitCompactInBetween() throws Exception { setup(2, 60 * 1024, false); final int NUMBER_OF_RECORDS = 1; SimpleIDGenerator idGen = new SimpleIDGenerator(1000); createJournal(); journal.setAutoReclaim(false); startJournal(); load(); for (int i = 0; i < NUMBER_OF_RECORDS; i++) { long tx = idGen.generateID(); addTx(tx, idGen.generateID()); journal.testCompact(); journal.testCompact(); journal.testCompact(); journal.testCompact(); logger.info("going to commit"); commit(tx); } stopJournal(); createJournal(); startJournal(); loadAndCheck(); }
@Test public void testCompactSecondFileReclaimed() throws Exception { setup(2, 60 * 1024, false); createJournal(); startJournal(); load(); addTx(1, 1, 2, 3, 4); journal.forceMoveNextFile(); addTx(1, 5, 6, 7, 8); commit(1); journal.forceMoveNextFile(); journal.testCompact(); add(10); stopJournal(); startJournal(); loadAndCheck(); stopJournal(); }
journal.appendAddRecord(2, (byte) 0, record, sync); try { startFlag.await(5, TimeUnit.SECONDS); journal.appendDeleteRecord(2, false); } catch (java.lang.IllegalStateException expected) { } catch (Exception e) { Assert.assertFalse(t.isAlive()); journal.flush();
@Test public void testMultipleAddUpdateDeleteDifferentRecordLengths() throws Exception { setup(10, 20480, true); createJournal(); startJournal(); load(); for (int i = 0; i < 100; i++) { byte[] record = generateRecord(RandomUtil.randomInterval(1500, 10000)); journal.appendAddRecord(i, (byte) 0, record, false); records.add(new RecordInfo(i, (byte) 0, record, false, (short) 0)); } for (int i = 0; i < 100; i++) { byte[] record = generateRecord(10 + RandomUtil.randomInterval(1500, 10000)); journal.appendUpdateRecord(i, (byte) 0, record, false); records.add(new RecordInfo(i, (byte) 0, record, true, (short) 0)); } for (int i = 0; i < 100; i++) { journal.appendDeleteRecord(i, false); removeRecordsForID(i); } stopJournal(); createJournal(); startJournal(); loadAndCheck(); stopJournal(); }
journal.forceMoveNextFile(); update(id); journal.forceMoveNextFile(); JournalFile[] files = journal.getDataFiles(); loadAndCheck(); journal.forceMoveNextFile(); JournalFile[] files2 = journal.getDataFiles(); journal.forceMoveNextFile(); JournalFile[] files3 = journal.getDataFiles(); loadAndCheck(); files3 = journal.getDataFiles();
@Test public void testDeleteWhileCleanup() throws Exception { setup(2, 60 * 1024, false); createJournal(); startJournal(); load(); for (int i = 0; i < 100; i++) { add(i); } journal.forceMoveNextFile(); for (int i = 10; i < 90; i++) { delete(i); } startCompact(); // Delete part of the live records while cleanup still working for (int i = 1; i < 5; i++) { delete(i); } finishCompact(); // Delete part of the live records after cleanup is done for (int i = 5; i < 10; i++) { delete(i); } assertEquals(9, journal.getCurrentFile().getNegCount(journal.getDataFiles()[0])); journal.forceMoveNextFile(); stopJournal(); createJournal(); startJournal(); loadAndCheck(); }
add(i); if (i % 10 == 0 && i > 0) { journal.forceMoveNextFile(); journal.forceMoveNextFile(); journal.forceMoveNextFile(); journal.forceMoveNextFile(); journal.forceMoveNextFile(); checkAndReclaimFiles(); Assert.assertEquals(0, journal.getDataFilesCount()); loadAndCheck(); Assert.assertEquals(0, journal.getDataFilesCount());
protected void addWithSize(final int size, final long... arguments) throws Exception { for (long element : arguments) { byte[] record = generateRecord(size); beforeJournalOperation(); journal.appendAddRecord(element, (byte) 0, record, sync); records.add(new RecordInfo(element, (byte) 0, record, false, (short) 0)); } journal.debugWait(); }