private int drainStreams(LogReader reader0, int num0, LogReader reader1, int num1) throws Exception { // Allow time for watches to fire Thread.sleep(15); int numTrans = 0; LogRecord record; int i = 0; while (i < num0) { record = reader0.readNext(false); if (null != record) { assertTrue((record.getTransactionId() % 2 == 0)); DLMTestUtil.verifyLogRecord(record); numTrans++; i++; LOG.info("Read record {}", record); } } i = 0; while (i < num1) { record = reader1.readNext(false); if (null != record) { assertTrue((record.getTransactionId() % 2 == 1)); DLMTestUtil.verifyLogRecord(record); numTrans++; i++; LOG.info("Read record {}", record); } } return numTrans; }
static long getNumberofLogRecords(DistributedLogManager bkdlm, long startTxId) throws IOException { long numLogRecs = 0; LogReader reader = bkdlm.getInputStream(startTxId); LogRecord record = reader.readNext(false); while (null != record) { numLogRecs++; verifyLogRecord(record); record = reader.readNext(false); } reader.close(); return numLogRecs; }
LogRecordWithDLSN record = Await.result(reader.readNext()); assertEquals(1L, record.getTransactionId()); DLMTestUtil.verifyLogRecord(record);
private void verifyEntries(String name, long readFromTxId, long startTxId, int numEntries) throws Exception { DistributedLogManager dlm = createNewDLM(conf, name); LogReader reader = dlm.getInputStream(readFromTxId); long txid = startTxId; int numRead = 0; LogRecord r = reader.readNext(false); while (null != r) { DLMTestUtil.verifyLogRecord(r); LOG.trace("Read entry {}.", r.getTransactionId()); assertEquals(txid++, r.getTransactionId()); ++numRead; r = reader.readNext(false); } assertEquals(numEntries, numRead); reader.close(); dlm.close(); }
@Test(timeout = 15000) public void testHandleInconsistentMetadataDLSNNonBlocking() throws Exception { String name = "distrlog-inconsistent-metadata-nonblocking-read-dlsn"; long numRecordsWritten = createStreamWithInconsistentMetadata(name); DistributedLogManager dlm = createNewDLM(conf, name); try { LogReader reader = dlm.getInputStream(DLSN.InitialDLSN); long numRecordsRead = 0; long lastTxId = -1; while (numRecordsRead < numRecordsWritten) { LogRecord record = reader.readNext(false); if (record != null) { DLMTestUtil.verifyLogRecord(record); Assert.assertTrue(lastTxId < record.getTransactionId()); lastTxId = record.getTransactionId(); numRecordsRead++; } else { Thread.sleep(1); } } reader.close(); } finally { dlm.close(); } } }
@Test(timeout = 15000) public void testHandleInconsistentMetadataNonBlocking() throws Exception { String name = "distrlog-inconsistent-metadata-nonblocking-read"; long numRecordsWritten = createStreamWithInconsistentMetadata(name); DistributedLogManager dlm = createNewDLM(conf, name); try { LogReader reader = dlm.getInputStream(45); long numRecordsRead = 0; long lastTxId = -1; while (numRecordsRead < (numRecordsWritten / 2)) { LogRecord record = reader.readNext(false); if (record != null) { DLMTestUtil.verifyLogRecord(record); Assert.assertTrue(lastTxId < record.getTransactionId()); lastTxId = record.getTransactionId(); numRecordsRead++; } else { Thread.sleep(1); } } reader.close(); } finally { dlm.close(); } }
@Test(timeout = 60000) public void testHandleInconsistentMetadata() throws Exception { String name = "distrlog-inconsistent-metadata-blocking-read"; long numRecordsWritten = createStreamWithInconsistentMetadata(name); DistributedLogManager dlm = createNewDLM(conf, name); try { LogReader reader = dlm.getInputStream(45); long numRecordsRead = 0; LogRecord record = reader.readNext(false); long lastTxId = -1; while (numRecordsRead < numRecordsWritten / 2) { if (null != record) { DLMTestUtil.verifyLogRecord(record); Assert.assertTrue(lastTxId < record.getTransactionId()); lastTxId = record.getTransactionId(); numRecordsRead++; } else { Thread.sleep(1); } record = reader.readNext(false); } reader.close(); assertEquals(numRecordsWritten / 2, numRecordsRead); } finally { dlm.close(); } }
long expectedTxId = 1; while (null != record) { DLMTestUtil.verifyLogRecord(record); assertEquals(expectedTxId, record.getTransactionId()); expectedTxId++;
LogRecord record = reader.readNext(false); assert (null != record); DLMTestUtil.verifyLogRecord(record); long lastTxId = record.getTransactionId(); record = reader.readNext(false); while (null != record) { DLMTestUtil.verifyLogRecord(record); assert (lastTxId < record.getTransactionId()); lastTxId = record.getTransactionId();
@Test(timeout = 60000) public void testWriteFailsAfterMarkEndOfStream() throws Exception { String name = "distrlog-mark-end-failure"; DistributedLogManager dlm = createNewDLM(conf, name); long txid = 1; txid = writeAndMarkEndOfStream(dlm, txid); assertEquals(txid - 1, dlm.getLastTxId()); LogRecord last = dlm.getLastLogRecord(); assertEquals(txid - 1, last.getTransactionId()); DLMTestUtil.verifyLogRecord(last); assert(dlm.isEndOfStreamMarked()); LogWriter writer = null; boolean exceptionEncountered = false; try { writer = dlm.startLogSegmentNonPartitioned(); for (long j = 1; j <= DEFAULT_SEGMENT_SIZE / 2; j++) { writer.write(DLMTestUtil.getLogRecordInstance(txid++)); } } catch (EndOfStreamException exc) { exceptionEncountered = true; } writer.close(); assert(exceptionEncountered); }
@Test(timeout = 60000) public void testReaderLockIfLockPathDoesntExist() throws Exception { final String name = runtime.getMethodName(); DistributedLogManager dlm = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.closeAndComplete(); Future<AsyncLogReader> futureReader1 = dlm.getAsyncLogReaderWithLock(DLSN.InitialDLSN); BKAsyncLogReaderDLSN reader1 = (BKAsyncLogReaderDLSN) Await.result(futureReader1); LogRecordWithDLSN record = Await.result(reader1.readNext()); assertEquals(1L, record.getTransactionId()); assertEquals(0L, record.getSequenceId()); DLMTestUtil.verifyLogRecord(record); String readLockPath = reader1.bkLedgerManager.getReadLockPath(); Utils.close(reader1); // simulate a old stream created without readlock path writer.bkDistributedLogManager.getWriterZKC().get().delete(readLockPath, -1); Future<AsyncLogReader> futureReader2 = dlm.getAsyncLogReaderWithLock(DLSN.InitialDLSN); AsyncLogReader reader2 = Await.result(futureReader2); record = Await.result(reader2.readNext()); assertEquals(1L, record.getTransactionId()); assertEquals(0L, record.getSequenceId()); DLMTestUtil.verifyLogRecord(record); }
assertEquals(2L, bulkReadRecords.get(1).getTransactionId()); for (LogRecordWithDLSN record : bulkReadRecords) { DLMTestUtil.verifyLogRecord(record); DLMTestUtil.verifyLogRecord(record);
assertEquals(1L, bulkReadRecords.get(0).getTransactionId()); for (LogRecordWithDLSN record : bulkReadRecords) { DLMTestUtil.verifyLogRecord(record); DLMTestUtil.verifyLogRecord(record);
long lastTxId = -1; while (null != record) { DLMTestUtil.verifyLogRecord(record); assert (lastTxId < record.getTransactionId()); lastTxId = record.getTransactionId();
assert (lastTxId < record.getTransactionId()); lastTxId = record.getTransactionId(); DLMTestUtil.verifyLogRecord(record); numTrans++;
++numReads; assertEquals(expectedTxId, record.getTransactionId()); DLMTestUtil.verifyLogRecord(record); ++expectedTxId; record = reader.readNext(false);
long lastTxId = -1; while (null != record) { DLMTestUtil.verifyLogRecord(record); assert (lastTxId < record.getTransactionId()); lastTxId = record.getTransactionId();
LogRecord last = dlm.getLastLogRecord(); assertEquals(txid - 1, last.getTransactionId()); DLMTestUtil.verifyLogRecord(last); assertEquals(txid - 1, dlm.getLogRecordCount());
LogRecord record = reader.readNext(false); while (null != record) { DLMTestUtil.verifyLogRecord(record); numTrans++; record = reader.readNext(false);
@Test(timeout = 60000) public void testReadingFromEmptyLog() throws Exception { String name = testName.getMethodName(); DistributedLogConfiguration confLocal = new DistributedLogConfiguration(); confLocal.addConfiguration(conf); confLocal.setOutputBufferSize(0); confLocal.setPeriodicFlushFrequencyMilliSeconds(Integer.MAX_VALUE); DistributedLogManager dlm = createNewDLM(confLocal, name); BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned(); // write a record but not commit LogRecord op = DLMTestUtil.getLogRecordInstance(1L); out.write(op); LogReader reader = dlm.getInputStream(1L); assertNull(reader.readNext(true)); assertNull(reader.readNext(false)); op = DLMTestUtil.getLogRecordInstance(2L); out.write(op); // reader is able to read first record LogRecord record = waitForNextRecord(reader); assertNotNull(record); assertEquals(1L, record.getTransactionId()); DLMTestUtil.verifyLogRecord(record); assertNull(reader.readNext(true)); out.close(); reader.close(); dlm.close(); }