/** * Flushes all the data up to this point, * adds the end of stream marker and marks the stream * as read-only in the metadata. No appends to the * stream will be allowed after this point */ @Override public void markEndOfStream() throws IOException { FutureUtils.result(getLedgerWriter(DistributedLogConstants.MAX_TXID, true).markEndOfStream()); closeAndComplete(); }
public static void generateCompletedLogSegments(DistributedLogManager manager, DistributedLogConfiguration conf, long numCompletedSegments, long segmentSize) throws Exception { BKDistributedLogManager dlm = (BKDistributedLogManager) manager; long txid = 1L; for (long i = 0; i < numCompletedSegments; i++) { BKSyncLogWriter writer = dlm.startLogSegmentNonPartitioned(); for (long j = 1; j <= segmentSize; j++) { writer.write(DLMTestUtil.getLogRecordInstance(txid++)); } writer.closeAndComplete(); } }
@Override public void run() { try { int txid = 1; for (long i = 0; i < numSegments; i++) { long start = txid; BKSyncLogWriter writer = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned(); for (long j = 1; j <= segmentSize; j++) { writer.write(DLMTestUtil.getLargeLogRecordInstance(txid++)); if ((i == 0) && (j == 1)) { latch.countDown(); } } writer.closeAndComplete(); Thread.sleep(100); } } catch (Exception exc) { if (!executor.isShutdown()) { currentThread.interrupt(); } } } }, 0, TimeUnit.MILLISECONDS);
writer.closeAndComplete(); if (!heartBeatUsingControlRecs) { Thread.sleep(2 * idleReaderErrorThreshold);
@Test(timeout = 60000) public void testSimpleWrite() throws Exception { BKDistributedLogManager dlm = createNewDLM(conf, "distrlog-simplewrite"); BKSyncLogWriter out = dlm.startLogSegmentNonPartitioned(); for (long i = 1; i <= 100; i++) { LogRecord op = DLMTestUtil.getLogRecordInstance(i); out.write(op); } BKLogSegmentWriter perStreamLogWriter = out.getCachedLogWriter(); out.closeAndComplete(); BKLogWriteHandler blplm = dlm.createWriteHandler(true); assertNotNull(zkc.exists(blplm.completedLedgerZNode(1, 100, perStreamLogWriter.getLogSegmentSequenceNumber()), false)); FutureUtils.result(blplm.asyncClose()); }
@Test(timeout = 60000) public void testNumberOfTransactions() throws Exception { String name = "distrlog-txncount"; DistributedLogManager dlm = createNewDLM(conf, name); BKSyncLogWriter out = (BKSyncLogWriter)dlm.startLogSegmentNonPartitioned(); for (long i = 1; i <= 100; i++) { LogRecord op = DLMTestUtil.getLogRecordInstance(i); out.write(op); } out.closeAndComplete(); long numTrans = DLMTestUtil.getNumberofLogRecords(createNewDLM(conf, name), 1); assertEquals(100, numTrans); dlm.close(); }
@Test(timeout = 60000) public void testSimpleRead() throws Exception { String name = "distrlog-simpleread"; DistributedLogManager dlm = createNewDLM(conf, name); final long numTransactions = 10000; BKSyncLogWriter out = (BKSyncLogWriter)dlm.startLogSegmentNonPartitioned(); for (long i = 1; i <= numTransactions; i++) { LogRecord op = DLMTestUtil.getLogRecordInstance(i); out.write(op); } out.closeAndComplete(); assertEquals(numTransactions, DLMTestUtil.getNumberofLogRecords(createNewDLM(conf, name), 1)); dlm.close(); }
@Test(timeout = 60000) public void testCreateReaderBeyondLastTransactionId() throws Exception { String name = testName.getMethodName(); DistributedLogManager dlm = createNewDLM(conf, name); BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned(); for (long i = 1; i < 10; i++) { LogRecord op = DLMTestUtil.getLogRecordInstance(i); out.write(op); } out.closeAndComplete(); LogReader reader = dlm.getInputStream(20L); assertNull(reader.readNext(false)); // write another 20 records out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned(); for (long i = 10; i < 30; i++) { LogRecord op = DLMTestUtil.getLogRecordInstance(i); out.write(op); } out.closeAndComplete(); for (int i = 0; i < 10; i++) { LogRecord record = waitForNextRecord(reader); assertEquals(20L + i, record.getTransactionId()); } assertNull(reader.readNext(false)); }
private long writeAndMarkEndOfStream(DistributedLogManager dlm, long txid) throws Exception { for (long i = 0; i < 3; i++) { long start = txid; BKSyncLogWriter writer = (BKSyncLogWriter)dlm.startLogSegmentNonPartitioned(); for (long j = 1; j <= DEFAULT_SEGMENT_SIZE; j++) { writer.write(DLMTestUtil.getLogRecordInstance(txid++)); } BKLogSegmentWriter perStreamLogWriter = writer.getCachedLogWriter(); if (i < 2) { writer.closeAndComplete(); BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true); assertNotNull(zkc.exists(blplm.completedLedgerZNode(start, txid - 1, perStreamLogWriter.getLogSegmentSequenceNumber()), false)); FutureUtils.result(blplm.asyncClose()); } else { writer.markEndOfStream(); BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true); assertNotNull(zkc.exists(blplm.completedLedgerZNode(start, DistributedLogConstants.MAX_TXID, perStreamLogWriter.getLogSegmentSequenceNumber()), false)); FutureUtils.result(blplm.asyncClose()); } } return txid; }
out.closeAndComplete(); BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true); assertNotNull(
@Test(timeout = 60000) public void testDeletingLogWhileReading() throws Exception { String name = testName.getMethodName(); DistributedLogManager dlm = createNewDLM(conf, name); BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned(); for (long i = 1; i < 10; i++) { LogRecord op = DLMTestUtil.getLogRecordInstance(i); out.write(op); } out.closeAndComplete(); LogReader reader = dlm.getInputStream(1L); for (int i = 1; i < 10; i++) { LogRecord record = waitForNextRecord(reader); assertEquals((long) i, record.getTransactionId()); } DistributedLogManager deleteDLM = createNewDLM(conf, name); deleteDLM.delete(); LogRecord record; try { record = reader.readNext(false); while (null == record) { record = reader.readNext(false); } fail("Should fail reading next with LogNotFound"); } catch (LogNotFoundException lnfe) { // expected } }
(BKSyncLogWriter) dlm.startLogSegmentNonPartitioned(); writer.write(DLMTestUtil.getLogRecordInstance(31L)); writer.closeAndComplete();
out.write(op); out.closeAndComplete();
writer.closeAndComplete(); BKLogWriteHandler blplm = dlm.createWriteHandler(true); assertNotNull(zkc.exists(blplm.completedLedgerZNode(start, txid - 1,
writer.closeAndComplete(); BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true); assertNotNull(zkc.exists(blplm.completedLedgerZNode(start, txid - 1,
@Test(timeout = 60000) public void testCompleteLogSegmentConflicts() throws Exception { URI uri = createURI(); String streamName = testName.getMethodName(); DistributedLogConfiguration conf = new DistributedLogConfiguration() .setLockTimeout(99999) .setOutputBufferSize(0) .setImmediateFlushEnabled(true) .setEnableLedgerAllocatorPool(true) .setLedgerAllocatorPoolName("test"); DistributedLogNamespace namespace = DistributedLogNamespaceBuilder.newBuilder().conf(conf).uri(uri).build(); namespace.createLog(streamName); DistributedLogManager dlm1 = namespace.openLog(streamName); DistributedLogManager dlm2 = namespace.openLog(streamName); // dlm1 is writing BKSyncLogWriter out1 = (BKSyncLogWriter) dlm1.startLogSegmentNonPartitioned(); out1.write(DLMTestUtil.getLogRecordInstance(1)); // before out1 complete, out2 is in on recovery // it completed the log segments which bump the version of /ledgers znode BKAsyncLogWriter out2 = (BKAsyncLogWriter) dlm2.startAsyncLogSegmentNonPartitioned(); try { out1.closeAndComplete(); fail("Should fail closeAndComplete since other people already completed it."); } catch (IOException ioe) { } } }
out.closeAndComplete(); BKLogWriteHandler blplm = ((BKDistributedLogManager) (dlm)).createWriteHandler(true);
out.closeAndComplete(); BKLogWriteHandler blplm = dlm.createWriteHandler(true);
/** * Create Log Segment for an pre-create stream. No max ledger sequence number recorded. */ @Test(timeout = 60000) public void testCreateLogSegmentOnPrecreatedStream() throws Exception { URI uri = createURI(); String streamName = testName.getMethodName(); DistributedLogConfiguration conf = new DistributedLogConfiguration() .setLockTimeout(99999) .setOutputBufferSize(0) .setImmediateFlushEnabled(true) .setEnableLedgerAllocatorPool(true) .setLedgerAllocatorPoolName("test"); BKDistributedLogNamespace namespace = BKDistributedLogNamespace.newBuilder().conf(conf).uri(uri).build(); namespace.createLog(streamName); MaxLogSegmentSequenceNo max1 = getMaxLogSegmentSequenceNo(namespace.getSharedWriterZKCForDL(), uri, streamName, conf); assertEquals(DistributedLogConstants.UNASSIGNED_LOGSEGMENT_SEQNO, max1.getSequenceNumber()); DistributedLogManager dlm = namespace.openLog(streamName); final int numSegments = 3; for (int i = 0; i < numSegments; i++) { BKSyncLogWriter out = (BKSyncLogWriter) dlm.startLogSegmentNonPartitioned(); out.write(DLMTestUtil.getLogRecordInstance(i)); out.closeAndComplete(); } MaxLogSegmentSequenceNo max2 = getMaxLogSegmentSequenceNo(namespace.getSharedWriterZKCForDL(), uri, streamName, conf); assertEquals(3, max2.getSequenceNumber()); dlm.close(); namespace.close(); }