Refine search
private void setupJournal(final int journalSize, final int alignment, final int numberOfMinimalFiles) throws Exception { if (factory == null) { factory = new FakeSequentialFileFactory(alignment, true); } if (journalImpl != null) { journalImpl.stop(); } journalImpl = new JournalImpl(journalSize, numberOfMinimalFiles, numberOfMinimalFiles, 0, 0, factory, "tt", "tt", 1000); journalImpl.start(); records.clear(); transactions.clear(); journalImpl.load(records, transactions, null); }
private static void addJournal(File folder, boolean crash) throws Exception { JournalImpl journal = createJournal(folder, crash); journal.loadInternalOnly(); for (int i = 0; i < 1000; i++) { journal.appendAddRecord(i, (byte) 1, new byte[5], true); } for (int i = 0; i < 100; i++) { journal.appendDeleteRecord(i, true); } journal.compact(); journal.stop(); }
public static void exportJournal(final String directory, final String journalPrefix, final String journalSuffix, final int minFiles, final int fileSize, final PrintStream out) throws Exception { NIOSequentialFileFactory nio = new NIOSequentialFileFactory(new File(directory), null, 1); JournalImpl journal = new JournalImpl(fileSize, minFiles, minFiles, 0, 0, nio, journalPrefix, journalSuffix, 1); List<JournalFile> files = journal.orderFiles(); for (JournalFile file : files) { out.println("#File," + file); exportJournalFile(out, nio, file); } }
private void compactJournal(final File directory, final String journalPrefix, final String journalSuffix, final int minFiles, final int fileSize, final IOCriticalErrorListener listener) throws Exception { NIOSequentialFileFactory nio = new NIOSequentialFileFactory(directory, listener, 1); JournalImpl journal = new JournalImpl(fileSize, minFiles, minFiles, 0, 0, nio, journalPrefix, journalSuffix, 1); journal.start(); journal.loadInternalOnly(); journal.compact(); journal.stop(); } }
JournalImpl journal = new JournalImpl(fileSize, minFiles, minFiles, 0, 0, nio, journalPrefix, journalSuffix, 1); if (journal.orderFiles().size() != 0) { throw new IllegalStateException("Import needs to create a brand new journal"); journal.start(); journal.loadInternalOnly(); journal.appendAddRecord(info.id, info.userRecordType, info.data, false); } else if (operation.equals("AddRecordTX")) { long txID = parseLong("txID", lineProperties); counter.incrementAndGet(); RecordInfo info = parseRecord(lineProperties); journal.appendAddRecordTransactional(txID, info.id, info.userRecordType, info.data); } else if (operation.equals("UpdateTX")) { long txID = parseLong("txID", lineProperties); counter.incrementAndGet(); RecordInfo info = parseRecord(lineProperties); journal.appendUpdateRecordTransactional(txID, info.id, info.userRecordType, info.data); } else if (operation.equals("Update")) { RecordInfo info = parseRecord(lineProperties); journal.appendUpdateRecord(info.id, info.userRecordType, info.data, false); } else if (operation.equals("DeleteRecord")) { long id = parseLong("id", lineProperties); journal.appendDeleteRecord(id, false);
public static void describeBindingsJournal(final File bindingsDir, PrintStream out, boolean safe) throws Exception { SequentialFileFactory bindingsFF = new NIOSequentialFileFactory(bindingsDir, null, 1); JournalImpl bindings = new JournalImpl(1024 * 1024, 2, 2, -1, 0, bindingsFF, "activemq-bindings", "bindings", 1); describeJournal(bindingsFF, bindings, bindingsDir, out, safe); }
journalImpl = new JournalImpl(512 + 512 * 3, 20, 20, 0, 0, factory, "amq", "amq", 1000); journalImpl.start(); journalImpl.load(AlignedJournalImplTest.dummyLoader); journalImpl.appendAddRecord(1L, (byte) 0, new SimpleEncoding(100, (byte) 'a'), false); journalImpl.appendAddRecord(2L, (byte) 0, new SimpleEncoding(100, (byte) 'b'), false); journalImpl.appendAddRecord(3L, (byte) 0, new SimpleEncoding(100, (byte) 'b'), false); journalImpl.appendAddRecord(4L, (byte) 0, new SimpleEncoding(100, (byte) 'b'), false); journalImpl.stop(); journalImpl = new JournalImpl(512 + 1024 + 512, 20, 20, 0, 0, factory, "amq", "amq", 1000); addActiveMQComponent(journalImpl); journalImpl.start(); journalImpl.load(AlignedJournalImplTest.dummyLoader); journalImpl.forceMoveNextFile(); journalImpl.appendDeleteRecord(1L, false); journalImpl.appendDeleteRecord(2L, false); journalImpl.appendDeleteRecord(3L, false); journalImpl.appendDeleteRecord(4L, false); journalImpl.stop(); journalImpl = new JournalImpl(512 + 1024 + 512, 20, 20, 0, 0, factory, "amq", "amq", 1000); addActiveMQComponent(journalImpl);
protected HashMap<Integer, AtomicInteger> countBindingJournal(Configuration config) throws Exception { final HashMap<Integer, AtomicInteger> recordsType = new HashMap<>(); SequentialFileFactory messagesFF = new NIOSequentialFileFactory(config.getBindingsLocation(), null, 1); JournalImpl messagesJournal = new JournalImpl(config.getJournalFileSize(), config.getJournalMinFiles(), config.getJournalPoolFiles(), 0, 0, messagesFF, "activemq-bindings", "bindings", 1); List<JournalFile> filesToRead = messagesJournal.orderFiles(); for (JournalFile file : filesToRead) { JournalImpl.readJournalFile(messagesFF, file, new RecordTypeCounter(recordsType)); } return recordsType; }
@Test public void testPartialDelete() throws Exception { final int JOURNAL_SIZE = 10000; setupAndLoadJournal(JOURNAL_SIZE, 100); journalImpl.setAutoReclaim(false); journalImpl.checkReclaimStatus(); journalImpl.debugWait(); Assert.assertEquals(2, factory.listFiles("tt").size()); UnitTestLogger.LOGGER.debug("Initial:--> " + journalImpl.debug()); UnitTestLogger.LOGGER.debug("_______________________________"); for (int i = 0; i < 50; i++) { journalImpl.appendAddRecord(i, (byte) 1, new SimpleEncoding(1, (byte) 'x'), false); } journalImpl.forceMoveNextFile(); // as the request to a new file is asynchronous, we need to make sure the // async requests are done journalImpl.debugWait(); Assert.assertEquals(3, factory.listFiles("tt").size()); for (int i = 10; i < 50; i++) { journalImpl.appendDeleteRecord(i, false); } journalImpl.debugWait(); setupAndLoadJournal(JOURNAL_SIZE, 100); Assert.assertEquals(10, records.size()); Assert.assertEquals(3, factory.listFiles("tt").size()); }
@Test public void testEmptyPrepare() throws Exception { final int JOURNAL_SIZE = 512 * 4; setupAndLoadJournal(JOURNAL_SIZE, 1); journalImpl.appendPrepareRecord(2L, new SimpleEncoding(10, (byte) 'j'), false); journalImpl.forceMoveNextFile(); journalImpl.appendAddRecord(1L, (byte) 0, new SimpleEncoding(10, (byte) 'k'), false); setupAndLoadJournal(JOURNAL_SIZE, 1); Assert.assertEquals(1, journalImpl.getDataFilesCount()); Assert.assertEquals(1, transactions.size()); journalImpl.forceMoveNextFile(); setupAndLoadJournal(JOURNAL_SIZE, 1); Assert.assertEquals(1, journalImpl.getDataFilesCount()); Assert.assertEquals(1, transactions.size()); journalImpl.appendCommitRecord(2L, false); journalImpl.appendDeleteRecord(1L, false); journalImpl.forceMoveNextFile(); setupAndLoadJournal(JOURNAL_SIZE, 0); journalImpl.forceMoveNextFile(); journalImpl.debugWait(); journalImpl.checkReclaimStatus(); Assert.assertEquals(0, transactions.size()); Assert.assertEquals(0, journalImpl.getDataFilesCount()); }
ArrayList<JournalFile> dataFilesToProcess = new ArrayList<>(filesRepository.getDataFilesCount()); boolean previousReclaimValue = isAutoReclaim(); onCompactStart(); onCompactLockingTheJournal(); setAutoReclaim(false); moveNextFile(false); for (final JournalFile file : dataFilesToProcess) { try { JournalImpl.readJournalFile(fileFactory, file, compactor, wholeFileBufferRef); } catch (Throwable e) { ActiveMQJournalLogger.LOGGER.compactReadError(file); onCompactDone(); SequentialFile controlFile = createControlFile(dataFilesToProcess, compactor.getNewDataFiles(), null); onCompactLockingTheJournal(); renameFiles(dataFilesToProcess, newDatafiles); deleteControlFile(controlFile); setAutoReclaim(previousReclaimValue);
journalImpl.appendAddRecordTransactional(1, i, (byte) 1, new SimpleEncoding(1, (byte) 1)); journalImpl.forceMoveNextFile(); journalImpl.appendCommitRecord(1L, false); journalImpl.debugWait(); journalImpl.checkReclaimStatus(); Assert.assertEquals(10, journalImpl.getDataFilesCount()); journalImpl.appendDeleteRecordTransactional(2L, i); journalImpl.forceMoveNextFile(); journalImpl.appendCommitRecord(2L, false); journalImpl.appendAddRecord(100, (byte) 1, new SimpleEncoding(5, (byte) 1), false); journalImpl.forceMoveNextFile(); journalImpl.appendAddRecord(101, (byte) 1, new SimpleEncoding(5, (byte) 1), false); journalImpl.checkReclaimStatus(); Assert.assertEquals(1, journalImpl.getDataFilesCount()); Assert.assertEquals(1, journalImpl.getDataFilesCount());
state != replicationSync; checkControlFile(wholeFileBufferRef); final List<JournalFile> orderedFiles = orderFiles(); int resultLastPost = JournalImpl.readJournalFile(fileFactory, file, new JournalReaderCallback() { setJournalState(JournalState.SYNCING); return new JournalLoadInformation(0, -1); setUpCurrentFile(lastDataPos); setJournalState(JournalState.LOADED); this.appendRollbackRecord(transaction.transactionID, false); checkReclaimStatus();
@Test public void testPrepareAloneOnSeparatedFile() throws Exception { final int JOURNAL_SIZE = 20000; setupAndLoadJournal(JOURNAL_SIZE, 100); Assert.assertEquals(0, records.size()); Assert.assertEquals(0, transactions.size()); for (int i = 0; i < 10; i++) { journalImpl.appendAddRecordTransactional(1L, i, (byte) 0, new SimpleEncoding(1, (byte) 15)); } journalImpl.forceMoveNextFile(); SimpleEncoding xidEncoding = new SimpleEncoding(10, (byte) 'a'); journalImpl.appendPrepareRecord(1L, xidEncoding, false); journalImpl.appendCommitRecord(1L, false); for (int i = 0; i < 10; i++) { journalImpl.appendDeleteRecordTransactional(2L, i); } journalImpl.appendCommitRecord(2L, false); journalImpl.appendAddRecord(100L, (byte) 0, new SimpleEncoding(1, (byte) 10), false); // Add // anything // to // keep // holding // the // file journalImpl.forceMoveNextFile(); journalImpl.checkReclaimStatus(); setupAndLoadJournal(JOURNAL_SIZE, 100); Assert.assertEquals(1, records.size()); }
private static JournalImpl createJournal(File folder, boolean crash) throws Exception { NIOSequentialFileFactory factory = new NIOSequentialFileFactory(folder, 10); JournalImpl journal = new JournalImpl(100 * 1024, 2, 2, 0, 0, factory, "jrntest", "jrn", 512) { @Override protected SequentialFile writeControlFile(final SequentialFileFactory fileFactory, final List<JournalFile> files, final List<JournalFile> newFiles, final List<Pair<String, String>> renames) throws Exception { if (crash) { SequentialFile controlFile = fileFactory.createSequentialFile(AbstractJournalUpdateTask.FILE_COMPACT_CONTROL); controlFile.open(); controlFile.close(); System.err.println("crashing after creation of control file"); System.exit(OK); } return JournalCompactor.writeControlFile(fileFactory, files, newFiles, renames); } }; journal.setAutoReclaim(false); journal.start(); return journal; }
public static JournalImpl createJournal(final String journalType, final String journalDir) { JournalImpl journal = new JournalImpl(10485760, 2, 2, 0, 0, ValidateTransactionHealthTest.getFactory(journalType, journalDir, 10485760), "journaltst", "tst", 500); return journal; }
journal.appendAddRecordTransactional(transactionId, id, (byte) 99, buffer.array()); journal.appendCommitRecord(transactionId, true); transactionCounter = 0; transactionId = nextID.incrementAndGet(); journal.appendAddRecord(id, (byte) 99, buffer.array(), false); journal.appendCommitRecord(transactionId, true); journal.debugWait();
journalImpl.appendAddRecord(10L, (byte) 0, new SimpleEncoding(10, (byte) 0), false); journalImpl.appendDeleteRecordTransactional(1L, 10L, new SimpleEncoding(100, (byte) 'j')); journalImpl.appendPrepareRecord(1, xid, false); journalImpl.debugWait(); journalImpl.appendCommitRecord(1L, false); journalImpl.debugWait();
@Test public void testReduceFreeFiles() throws Exception { final int JOURNAL_SIZE = 2000; setupAndLoadJournal(JOURNAL_SIZE, 100, 10); Assert.assertEquals(10, factory.listFiles("tt").size()); setupAndLoadJournal(JOURNAL_SIZE, 100, 2); Assert.assertEquals(10, factory.listFiles("tt").size()); for (int i = 0; i < 10; i++) { journalImpl.appendAddRecord(i, (byte) 0, new SimpleEncoding(1, (byte) 0), false); journalImpl.forceMoveNextFile(); } setupAndLoadJournal(JOURNAL_SIZE, 100, 2); Assert.assertEquals(10, records.size()); Assert.assertEquals(12, factory.listFiles("tt").size()); for (int i = 0; i < 10; i++) { journalImpl.appendDeleteRecord(i, false); } journalImpl.forceMoveNextFile(); journalImpl.checkReclaimStatus(); setupAndLoadJournal(JOURNAL_SIZE, 100, 2); Assert.assertEquals(0, records.size()); Assert.assertEquals(2, factory.listFiles("tt").size()); }
private void reload(final String type, final String journalDir, final long numberOfRecords, final int numberOfThreads) throws Exception { JournalImpl journal = ValidateTransactionHealthTest.createJournal(type, journalDir); journal.start(); try { Loader loadTest = new Loader(numberOfRecords); journal.load(loadTest); Assert.assertEquals(numberOfRecords * numberOfThreads, loadTest.numberOfAdds); Assert.assertEquals(0, loadTest.numberOfPreparedTransactions); Assert.assertEquals(0, loadTest.numberOfUpdates); Assert.assertEquals(0, loadTest.numberOfDeletes); if (loadTest.ex != null) { throw loadTest.ex; } } finally { journal.stop(); } }