private static LogRecordWithInputStream nextLogRecord(LogReader reader) throws IOException { LogRecordWithDLSN record = reader.readNext(false); if (null != record) { return new LogRecordWithInputStream(record); } else { record = reader.readNext(false); if (null != record) { return new LogRecordWithInputStream(record); } else { LOG.debug("No record"); return null; } } }
int countFromStartToEnd(DistributedLogManager dlm) throws Exception { int count = 0; try { LogReader reader = dlm.getInputStream(startDLSN); try { LogRecordWithDLSN record = reader.readNext(false); LogRecordWithDLSN preRecord = record; System.out.println("first record : " + record); while (null != record) { if (record.getDlsn().compareTo(endDLSN) > 0) { break; } ++count; if (count % 1000 == 0) { logger.info("read {} records from {}...", count, getStreamName()); } preRecord = record; record = reader.readNext(false); } System.out.println("last record : " + preRecord); } finally { reader.close(); } } finally { dlm.close(); } return count; }
static long getNumberofLogRecords(DistributedLogManager bkdlm, long startTxId) throws IOException { long numLogRecs = 0; LogReader reader = bkdlm.getInputStream(startTxId); LogRecord record = reader.readNext(false); while (null != record) { numLogRecs++; verifyLogRecord(record); record = reader.readNext(false); } reader.close(); return numLogRecs; }
@Override public void run() { while (running) { try { LogRecord r = reader.readNext(nonBlockReading); if (r != null) { readCount.incrementAndGet(); if (readCount.get() % 1000 == 0) { LOG.info("{} reading {}", getName(), r.getTransactionId()); } } } catch (DLInterruptedException die) { Thread.currentThread().interrupt(); } catch (IOException e) { break; } } }
private int drainStreams(LogReader reader0, int num0, LogReader reader1, int num1) throws Exception { // Allow time for watches to fire Thread.sleep(15); int numTrans = 0; LogRecord record; int i = 0; while (i < num0) { record = reader0.readNext(false); if (null != record) { assertTrue((record.getTransactionId() % 2 == 0)); DLMTestUtil.verifyLogRecord(record); numTrans++; i++; LOG.info("Read record {}", record); } } i = 0; while (i < num1) { record = reader1.readNext(false); if (null != record) { assertTrue((record.getTransactionId() % 2 == 1)); DLMTestUtil.verifyLogRecord(record); numTrans++; i++; LOG.info("Read record {}", record); } } return numTrans; }
private void verifyEntries(String name, long readFromTxId, long startTxId, int numEntries) throws Exception { DistributedLogManager dlm = createNewDLM(conf, name); LogReader reader = dlm.getInputStream(readFromTxId); long txid = startTxId; int numRead = 0; LogRecord r = reader.readNext(false); while (null != r) { DLMTestUtil.verifyLogRecord(r); LOG.trace("Read entry {}.", r.getTransactionId()); assertEquals(txid++, r.getTransactionId()); ++numRead; r = reader.readNext(false); } assertEquals(numEntries, numRead); reader.close(); dlm.close(); }
boolean exceptionEncountered = false; try { LogRecord record = reader.readNext(false); long lastTxId = -1; while (null != record) { lastTxId = record.getTransactionId(); numTrans++; record = reader.readNext(false); exceptionEncountered = false; try { reader.readNext(false); } catch (EndOfStreamException exc) { exceptionEncountered = true;
@Test(timeout = 60000) public void testHandleInconsistentMetadata() throws Exception { String name = "distrlog-inconsistent-metadata-blocking-read"; long numRecordsWritten = createStreamWithInconsistentMetadata(name); DistributedLogManager dlm = createNewDLM(conf, name); try { LogReader reader = dlm.getInputStream(45); long numRecordsRead = 0; LogRecord record = reader.readNext(false); long lastTxId = -1; while (numRecordsRead < numRecordsWritten / 2) { if (null != record) { DLMTestUtil.verifyLogRecord(record); Assert.assertTrue(lastTxId < record.getTransactionId()); lastTxId = record.getTransactionId(); numRecordsRead++; } else { Thread.sleep(1); } record = reader.readNext(false); } reader.close(); assertEquals(numRecordsWritten / 2, numRecordsRead); } finally { dlm.close(); } }
LogRecord record = reader.readNext(false); assert (null != record); DLMTestUtil.verifyLogRecord(record); record = reader.readNext(false); while (null != record) { DLMTestUtil.verifyLogRecord(record); lastTxId = record.getTransactionId(); numTrans++; record = reader.readNext(false); reader.readNext(false);
@Test(timeout = 15000) public void testHandleInconsistentMetadataNonBlocking() throws Exception { String name = "distrlog-inconsistent-metadata-nonblocking-read"; long numRecordsWritten = createStreamWithInconsistentMetadata(name); DistributedLogManager dlm = createNewDLM(conf, name); try { LogReader reader = dlm.getInputStream(45); long numRecordsRead = 0; long lastTxId = -1; while (numRecordsRead < (numRecordsWritten / 2)) { LogRecord record = reader.readNext(false); if (record != null) { DLMTestUtil.verifyLogRecord(record); Assert.assertTrue(lastTxId < record.getTransactionId()); lastTxId = record.getTransactionId(); numRecordsRead++; } else { Thread.sleep(1); } } reader.close(); } finally { dlm.close(); } }
@Test(timeout = 15000) public void testHandleInconsistentMetadataDLSNNonBlocking() throws Exception { String name = "distrlog-inconsistent-metadata-nonblocking-read-dlsn"; long numRecordsWritten = createStreamWithInconsistentMetadata(name); DistributedLogManager dlm = createNewDLM(conf, name); try { LogReader reader = dlm.getInputStream(DLSN.InitialDLSN); long numRecordsRead = 0; long lastTxId = -1; while (numRecordsRead < numRecordsWritten) { LogRecord record = reader.readNext(false); if (record != null) { DLMTestUtil.verifyLogRecord(record); Assert.assertTrue(lastTxId < record.getTransactionId()); lastTxId = record.getTransactionId(); numRecordsRead++; } else { Thread.sleep(1); } } reader.close(); } finally { dlm.close(); } } }
@Test(timeout = 60000) public void testDeletingLogWhileReading() throws Exception { String name = testName.getMethodName(); DistributedLogManager dlm = createNewDLM(conf, name); BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned(); for (long i = 1; i < 10; i++) { LogRecord op = DLMTestUtil.getLogRecordInstance(i); out.write(op); } out.closeAndComplete(); LogReader reader = dlm.getInputStream(1L); for (int i = 1; i < 10; i++) { LogRecord record = waitForNextRecord(reader); assertEquals((long) i, record.getTransactionId()); } DistributedLogManager deleteDLM = createNewDLM(conf, name); deleteDLM.delete(); LogRecord record; try { record = reader.readNext(false); while (null == record) { record = reader.readNext(false); } fail("Should fail reading next with LogNotFound"); } catch (LogNotFoundException lnfe) { // expected } }
@Test(timeout = 60000) public void testReadingFromEmptyLog() throws Exception { String name = testName.getMethodName(); DistributedLogConfiguration confLocal = new DistributedLogConfiguration(); confLocal.addConfiguration(conf); confLocal.setOutputBufferSize(0); confLocal.setPeriodicFlushFrequencyMilliSeconds(Integer.MAX_VALUE); DistributedLogManager dlm = createNewDLM(confLocal, name); BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned(); // write a record but not commit LogRecord op = DLMTestUtil.getLogRecordInstance(1L); out.write(op); LogReader reader = dlm.getInputStream(1L); assertNull(reader.readNext(true)); assertNull(reader.readNext(false)); op = DLMTestUtil.getLogRecordInstance(2L); out.write(op); // reader is able to read first record LogRecord record = waitForNextRecord(reader); assertNotNull(record); assertEquals(1L, record.getTransactionId()); DLMTestUtil.verifyLogRecord(record); assertNull(reader.readNext(true)); out.close(); reader.close(); dlm.close(); }
@Test(timeout = 60000) public void testCreateReaderBeyondLastTransactionId() throws Exception { String name = testName.getMethodName(); DistributedLogManager dlm = createNewDLM(conf, name); BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned(); for (long i = 1; i < 10; i++) { LogRecord op = DLMTestUtil.getLogRecordInstance(i); out.write(op); } out.closeAndComplete(); LogReader reader = dlm.getInputStream(20L); assertNull(reader.readNext(false)); // write another 20 records out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned(); for (long i = 10; i < 30; i++) { LogRecord op = DLMTestUtil.getLogRecordInstance(i); out.write(op); } out.closeAndComplete(); for (int i = 0; i < 10; i++) { LogRecord record = waitForNextRecord(reader); assertEquals(20L + i, record.getTransactionId()); } assertNull(reader.readNext(false)); }
@Test(timeout = 60000) public void testToolTruncateStream() throws Exception { DistributedLogManager dlm = DLMTestUtil.createNewDLM("testToolTruncateStream", conf, defaultUri); DLMTestUtil.generateCompletedLogSegments(dlm, conf, 3, 1000); DLSN dlsn = new DLSN(2,1,0); TruncateStreamCommand cmd = new TruncateStreamCommand(); cmd.setDlsn(dlsn); cmd.setUri(defaultUri); cmd.setStreamName("testToolTruncateStream"); cmd.setForce(true); assertEquals(0, cmd.runCmd()); LogReader reader = dlm.getInputStream(0); LogRecordWithDLSN record = reader.readNext(false); assertEquals(dlsn, record.getDlsn()); reader.close(); dlm.close(); } }