@Override public void close() throws IOException { logWriter.closeAndComplete(); }
writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.write(DLMTestUtil.getLogRecordInstance(2L)); writer.closeAndComplete();
BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.closeAndComplete();
@Test(timeout = 60000) public void testReaderLockCloseInAcquireCallback() throws Exception { final String name = runtime.getMethodName(); DistributedLogManager dlm = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.closeAndComplete(); final CountDownLatch latch = new CountDownLatch(1); Future<AsyncLogReader> futureReader1 = dlm.getAsyncLogReaderWithLock(DLSN.InitialDLSN); futureReader1.flatMap(new ExceptionalFunction<AsyncLogReader, Future<Void>>() { @Override public Future<Void> applyE(AsyncLogReader reader) throws IOException { return reader.asyncClose().map(new AbstractFunction1<Void, Void>() { @Override public Void apply(Void result) { latch.countDown(); return null; } }); } }); latch.await(); dlm.close(); }
slotIndex = 0; txIndex += batchSize; writer.closeAndComplete(); writer = (BKAsyncLogWriter)(dlm.startAsyncLogSegmentNonPartitioned()); checkAllSucceeded(writer, batchSize, recSize, ledgerIndex, entryIndex, slotIndex, txIndex); writer.closeAndComplete(); dlm.close();
static void writeRecordsForNonBlockingReads(DistributedLogConfiguration conf, DistributedLogManager dlm, boolean recover, long segmentSize) throws Exception { long txId = 1; for (long i = 0; i < 3; i++) { BKAsyncLogWriter writer = (BKAsyncLogWriter) dlm.startAsyncLogSegmentNonPartitioned(); for (long j = 1; j < segmentSize; j++) { FutureUtils.result(writer.write(DLMTestUtil.getLogRecordInstance(txId++))); } if (recover) { FutureUtils.result(writer.write(DLMTestUtil.getLogRecordInstance(txId++))); TimeUnit.MILLISECONDS.sleep(300); writer.abort(); LOG.debug("Recovering Segments"); BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true); FutureUtils.result(blplm.recoverIncompleteLogSegments()); FutureUtils.result(blplm.asyncClose()); LOG.debug("Recovered Segments"); } else { FutureUtils.result(writer.write(DLMTestUtil.getLogRecordInstance(txId++))); writer.closeAndComplete(); } TimeUnit.MILLISECONDS.sleep(300); } }
@Test(timeout = 60000) public void testOutstandingWriteLimitBlockAllLimitWithDarkmode() throws Exception { DistributedLogConfiguration confLocal = new DistributedLogConfiguration(); confLocal.addConfiguration(testConf); confLocal.setOutputBufferSize(0); confLocal.setImmediateFlushEnabled(true); confLocal.setPerWriterOutstandingWriteLimit(0); confLocal.setOutstandingWriteLimitDarkmode(true); DistributedLogManager dlm = createNewDLM(confLocal, runtime.getMethodName()); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm.startAsyncLogSegmentNonPartitioned()); ArrayList<Future<DLSN>> results = new ArrayList<Future<DLSN>>(1000); for (int i = 0; i < 1000; i++) { results.add(writer.write(DLMTestUtil.getLogRecordInstance(1L))); } for (Future<DLSN> result : results) { Await.result(result); } writer.closeAndComplete(); dlm.close(); }
writer.closeAndComplete(); LOG.info("Completed first log segment"); writer.closeAndComplete(); LOG.info("Completed third log segment");
writer.closeAndComplete(); fail("Should fail to complete a log segment when its ledger is fenced"); } catch (IOException ioe) {
writer.closeAndComplete(); fail("Should fail to complete a log segment when its ledger is fenced"); } catch (BKTransmitException bkte) {
/** * Test Case: A large write batch will span multiple packets. * @throws Exception */ @Test(timeout = 60000) public void testAsyncBulkWriteSpanningPackets() throws Exception { String name = "distrlog-testAsyncBulkWriteSpanningPackets"; DistributedLogConfiguration confLocal = new DistributedLogConfiguration(); confLocal.loadConf(testConf); confLocal.setOutputBufferSize(1024); DistributedLogManager dlm = createNewDLM(confLocal, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm.startAsyncLogSegmentNonPartitioned()); // First entry. int numTransmissions = 4; int recSize = 10*1024; int batchSize = (numTransmissions*MAX_LOGRECORDSET_SIZE+1)/recSize; long ledgerIndex = 1; long entryIndex = 0; long slotIndex = 0; long txIndex = 1; DLSN dlsn = checkAllSucceeded(writer, batchSize, recSize, ledgerIndex, entryIndex, slotIndex, txIndex); assertEquals(4, dlsn.getEntryId()); assertEquals(1, dlsn.getLogSegmentSequenceNo()); writer.closeAndComplete(); dlm.close(); }
writer.closeAndComplete(); dlm.close();
@Test(timeout = 60000) public void testReaderLockSharedDlmDoesNotConflict() throws Exception { String name = runtime.getMethodName(); DistributedLogManager dlm0 = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm0.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.write(DLMTestUtil.getLogRecordInstance(2L)); writer.closeAndComplete(); DistributedLogManager dlm1 = createNewDLM(conf, name); Future<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN); Future<AsyncLogReader> futureReader2 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN); // Both use the same client id, so there's no lock conflict. Not necessarily ideal, but how the // system currently works. Await.result(futureReader1); Await.result(futureReader2); dlm0.close(); dlm1.close(); }
@Test(timeout = 60000) public void testReaderLockFutureCancelledWhileLocked() throws Exception { String name = runtime.getMethodName(); DistributedLogManager dlm0 = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm0.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.write(DLMTestUtil.getLogRecordInstance(2L)); writer.closeAndComplete(); DistributedLogManager dlm1 = createNewDLM(conf, name); Future<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN); // Must not throw or cancel or do anything bad, future already completed. Await.result(futureReader1); FutureUtils.cancel(futureReader1); AsyncLogReader reader1 = Await.result(futureReader1); Await.result(reader1.readNext()); dlm0.close(); dlm1.close(); }
@Test(timeout = 60000) public void testReaderLockDlmClosed() throws Exception { String name = runtime.getMethodName(); DistributedLogManager dlm0 = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm0.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.write(DLMTestUtil.getLogRecordInstance(2L)); writer.closeAndComplete(); DistributedLogManager dlm1 = createNewDLM(conf, name); Future<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN); Await.result(futureReader1); BKDistributedLogManager dlm2 = (BKDistributedLogManager) createNewDLM(conf, name); Future<AsyncLogReader> futureReader2 = dlm2.getAsyncLogReaderWithLock(DLSN.InitialDLSN); dlm2.close(); try { Await.result(futureReader2); fail("should have thrown exception!"); } catch (LockClosedException ex) { } catch (LockCancelledException ex) { } dlm0.close(); dlm1.close(); }
@Test(timeout = 60000) public void testReaderLockSessionExpires() throws Exception { String name = runtime.getMethodName(); DistributedLogManager dlm0 = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm0.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.write(DLMTestUtil.getLogRecordInstance(2L)); writer.closeAndComplete(); DistributedLogManager dlm1 = createNewDLM(conf, name); Future<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN); AsyncLogReader reader1 = Await.result(futureReader1); ZooKeeperClientUtils.expireSession(((BKDistributedLogManager)dlm1).getWriterZKC(), zkServers, 1000); // The result of expireSession is somewhat non-deterministic with this lock. // It may fail with LockingException or it may succesfully reacquire, so for // the moment rather than make it deterministic we accept either result. boolean success = false; try { Await.result(reader1.readNext()); success = true; } catch (LockingException ex) { } if (success) { Await.result(reader1.readNext()); } Utils.close(reader1); dlm0.close(); dlm1.close(); }
@Test(timeout = 60000) public void testReaderLockFutureCancelledWhileWaiting() throws Exception { String name = runtime.getMethodName(); DistributedLogManager dlm0 = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm0.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.write(DLMTestUtil.getLogRecordInstance(2L)); writer.closeAndComplete(); DistributedLogManager dlm1 = createNewDLM(conf, name); Future<AsyncLogReader> futureReader1 = dlm1.getAsyncLogReaderWithLock(DLSN.InitialDLSN); AsyncLogReader reader1 = Await.result(futureReader1); DistributedLogManager dlm2 = createNewDLM(conf, name); Future<AsyncLogReader> futureReader2 = dlm2.getAsyncLogReaderWithLock(DLSN.InitialDLSN); try { FutureUtils.cancel(futureReader2); Await.result(futureReader2); fail("Should fail getting log reader as it is cancelled"); } catch (LockClosedException ex) { } catch (LockCancelledException ex) { } catch (OwnershipAcquireFailedException oafe) { } futureReader2 = dlm2.getAsyncLogReaderWithLock(DLSN.InitialDLSN); Utils.close(reader1); Await.result(futureReader2); dlm0.close(); dlm1.close(); dlm2.close(); }
@Test(timeout = 60000) public void testAsyncWritePendingWritesAbortedWhenLedgerRollTriggerFails() throws Exception { String name = runtime.getMethodName(); DistributedLogConfiguration confLocal = new DistributedLogConfiguration(); confLocal.loadConf(testConf); confLocal.setOutputBufferSize(1024); confLocal.setMaxLogSegmentBytes(1024); confLocal.setLogSegmentRollingIntervalMinutes(0); DistributedLogManager dlm = createNewDLM(confLocal, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm.startAsyncLogSegmentNonPartitioned()); // Write one record larger than max seg size. Ledger doesn't roll until next write. int txid = 1; LogRecord record = DLMTestUtil.getLogRecordInstance(txid++, 2048); Future<DLSN> result = writer.write(record); DLSN dlsn = Await.result(result, Duration.fromSeconds(10)); assertEquals(1, dlsn.getLogSegmentSequenceNo()); record = DLMTestUtil.getLogRecordInstance(txid++, MAX_LOGRECORD_SIZE + 1); result = writer.write(record); validateFutureFailed(result, LogRecordTooLongException.class); record = DLMTestUtil.getLogRecordInstance(txid++, MAX_LOGRECORD_SIZE + 1); result = writer.write(record); validateFutureFailed(result, WriteException.class); record = DLMTestUtil.getLogRecordInstance(txid++, MAX_LOGRECORD_SIZE + 1); result = writer.write(record); validateFutureFailed(result, WriteException.class); writer.closeAndComplete(); dlm.close(); }
@Test(timeout = 60000) public void testReaderLockIfLockPathDoesntExist() throws Exception { final String name = runtime.getMethodName(); DistributedLogManager dlm = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.closeAndComplete(); Future<AsyncLogReader> futureReader1 = dlm.getAsyncLogReaderWithLock(DLSN.InitialDLSN); BKAsyncLogReaderDLSN reader1 = (BKAsyncLogReaderDLSN) Await.result(futureReader1); LogRecordWithDLSN record = Await.result(reader1.readNext()); assertEquals(1L, record.getTransactionId()); assertEquals(0L, record.getSequenceId()); DLMTestUtil.verifyLogRecord(record); String readLockPath = reader1.bkLedgerManager.getReadLockPath(); Utils.close(reader1); // simulate a old stream created without readlock path writer.bkDistributedLogManager.getWriterZKC().get().delete(readLockPath, -1); Future<AsyncLogReader> futureReader2 = dlm.getAsyncLogReaderWithLock(DLSN.InitialDLSN); AsyncLogReader reader2 = Await.result(futureReader2); record = Await.result(reader2.readNext()); assertEquals(1L, record.getTransactionId()); assertEquals(0L, record.getSequenceId()); DLMTestUtil.verifyLogRecord(record); }