@Override public void process(LogRecordWithDLSN record) { if (found) { return; } this.result = record; if (record.getTransactionId() >= txId) { found = true; } }
@Override public Long apply(LogRecordWithDLSN record) { return record.getTransactionId(); } };
long getOffset() { return logRecord.getTransactionId() - logRecord.getPayload().length; } }
@Override public void onSuccess(LogRecordWithDLSN record) { this.lastSeenDLSN = record.getDlsn(); if (!startTransactionId.isPresent() || record.getTransactionId() >= startTransactionId.get()) { readAheadRecords.add(record); } if (readAheadRecords.size() >= maxNumCachedRecords) { setReadAheadCallback(this); } else { scheduleReadNext(); } }
/** * Find the id of the last edit log transaction written to a edit log * ledger. */ protected Pair<Long, DLSN> readLastTxIdInLedger(LogSegmentMetadata l) throws IOException { LogRecordWithDLSN record = recoverLastRecordInLedger(l, false, false, true); if (null == record) { return Pair.of(DistributedLogConstants.EMPTY_LOGSEGMENT_TX_ID, DLSN.InvalidDLSN); } else { return Pair.of(record.getTransactionId(), record.getDlsn()); } }
@Override public void onSuccess(LogRecordWithDLSN record) { System.out.println("Received record " + record.getDlsn()); System.out.println("\"\"\""); System.out.println(new String(record.getPayload(), UTF_8)); System.out.println("\"\"\""); long diffInMilliseconds = System.currentTimeMillis() - record.getTransactionId(); if (!caughtup.get() && diffInMilliseconds < 2000) { System.out.println("Reader caught with latest data"); caughtup.set(true); } reader.readNext().addEventListener(this); } };
public long getLastTxId(boolean recover, boolean includeEndOfStream) throws IOException { checkLogStreamExists(); return getLastLogRecord(recover, includeEndOfStream).getTransactionId(); }
LogRecordWithInputStream(LogRecordWithDLSN logRecord) { Preconditions.checkNotNull(logRecord); LOG.debug("Got record dlsn = {}, txid = {}, len = {}", new Object[] {logRecord.getDlsn(), logRecord.getTransactionId(), logRecord.getPayload().length}); this.logRecord = logRecord; this.payloadStream = logRecord.getPayLoadInputStream(); }
@Override public String toString() { return "LogRecordWithDLSN{" + "dlsn=" + dlsn + ", txid=" + getTransactionId() + ", position=" + getPositionWithinLogSegment() + ", isControl=" + isControl() + ", isEndOfStream=" + isEndOfStream() + '}'; } }
@Override public long getFirstTxId() throws IOException { checkClosedOrInError("getFirstTxId"); return FutureUtils.result(getFirstRecordAsyncInternal()).getTransactionId(); }
/** * Check if an end of stream marker was added to the stream * A stream with an end of stream marker cannot be appended to * * @return true if the marker was added to the stream, false otherwise */ @Override public boolean isEndOfStreamMarked() throws IOException { checkClosedOrInError("isEndOfStreamMarked"); long lastTxId = FutureUtils.result(getLastLogRecordAsyncInternal(false, true)).getTransactionId(); return lastTxId == DistributedLogConstants.MAX_TXID; }
@Override public int read(byte[] b, int off, int len) throws IOException { int read = 0; if (currentLogRecord == null) { currentLogRecord = nextLogRecord(); if (currentLogRecord == null) { return read; } } while (read < len) { int thisread = currentLogRecord.getPayLoadInputStream().read(b, off + read, (len - read)); if (thisread == -1) { currentLogRecord = nextLogRecord(); if (currentLogRecord == null) { return read; } } else { LOG.debug("Offset saved = {}, persisted = {}", currentPosition, currentLogRecord.getLogRecord().getTransactionId()); currentPosition += thisread; read += thisread; } } return read; }
public static Reader of(LogRecordWithDLSN record) throws IOException { Preconditions.checkArgument(record.isRecordSet(), "record is not a recordset"); byte[] data = record.getPayload(); DLSN dlsn = record.getDlsn(); int startPosition = record.getPositionWithinLogSegment(); long startSequenceId = record.getStartSequenceIdOfCurrentSegment(); return new EnvelopedRecordSetReader( dlsn.getLogSegmentSequenceNo(), dlsn.getEntryId(), record.getTransactionId(), dlsn.getSlotId(), startPosition, startSequenceId, new ByteArrayInputStream(data)); }
/** * Begin appending to the end of the log stream which is being treated as a sequence of bytes * * @return the writer interface to generate log records */ public AppendOnlyStreamWriter getAppendOnlyStreamWriter() throws IOException { long position; try { position = FutureUtils.result(getLastLogRecordAsyncInternal(true, false)).getTransactionId(); if (DistributedLogConstants.INVALID_TXID == position || DistributedLogConstants.EMPTY_LOGSEGMENT_TX_ID == position) { position = 0; } } catch (LogEmptyException ex) { position = 0; } catch (LogNotFoundException ex) { position = 0; } return new AppendOnlyStreamWriter(startAsyncLogSegmentNonPartitioned(), position); }
@Override public Future<LogSegmentMetadata> updateLastRecord(LogSegmentMetadata segment, LogRecordWithDLSN record) { DLSN dlsn = record.getDlsn(); Preconditions.checkState(!segment.isInProgress(), "Updating last dlsn for an inprogress log segment isn't supported."); Preconditions.checkArgument(segment.isDLSNinThisSegment(dlsn), "DLSN " + dlsn + " doesn't belong to segment " + segment); final LogSegmentMetadata newSegment = segment.mutator() .setLastDLSN(dlsn) .setLastTxId(record.getTransactionId()) .setRecordCount(record) .build(); return updateSegmentMetadata(newSegment); }
@Test(timeout = 60000) public void testGetLogRecordGreaterThanTxIdOnLargeSegment() throws Exception { String streamName = runtime.getMethodName(); BKDistributedLogManager bkdlm = createNewDLM(conf, streamName); DLMTestUtil.generateLogSegmentNonPartitioned(bkdlm, 0 /* control recs */, 100, 1L /* txid */, 3L); Optional<LogRecordWithDLSN> result = FutureUtils.result(getLogRecordNotLessThanTxId(bkdlm, 0, 23L)); assertTrue(result.isPresent()); assertEquals(25L, result.get().getTransactionId()); }
@Test(timeout = 60000) public void testGetLogRecordNotLessThanTxIdOnSmallSegment() throws Exception { String streamName = runtime.getMethodName(); BKDistributedLogManager bkdlm = createNewDLM(conf, streamName); DLMTestUtil.generateLogSegmentNonPartitioned(bkdlm, 0 /* control recs */, 5, 1L /* txid */); Optional<LogRecordWithDLSN> result = FutureUtils.result(getLogRecordNotLessThanTxId(bkdlm, 0, 3L)); assertTrue(result.isPresent()); assertEquals(3L, result.get().getTransactionId()); }
@Test(timeout = 60000) public void testGetLogRecordNotLessThanTxIdOnLargeSegment() throws Exception { String streamName = runtime.getMethodName(); BKDistributedLogManager bkdlm = createNewDLM(conf, streamName); DLMTestUtil.generateLogSegmentNonPartitioned(bkdlm, 0 /* control recs */, 100, 1L /* txid */); Optional<LogRecordWithDLSN> result = FutureUtils.result(getLogRecordNotLessThanTxId(bkdlm, 0, 9L)); assertTrue(result.isPresent()); assertEquals(9L, result.get().getTransactionId()); }
@Test(timeout = 60000) public void testGetLogRecordNotLessThanTxIdWithLessTxId() throws Exception { String streamName = runtime.getMethodName(); BKDistributedLogManager bkdlm = createNewDLM(conf, streamName); DLMTestUtil.generateLogSegmentNonPartitioned(bkdlm, 0 /* control recs */, 1, 999L /* txid */); Optional<LogRecordWithDLSN> result = FutureUtils.result(getLogRecordNotLessThanTxId(bkdlm, 0, 99L)); assertTrue(result.isPresent()); assertEquals(999L, result.get().getTransactionId()); assertEquals(0L, result.get().getDlsn().getEntryId()); assertEquals(0L, result.get().getDlsn().getSlotId()); }
@Test(timeout = 60000) public void testReaderLockIfLockPathDoesntExist() throws Exception { final String name = runtime.getMethodName(); DistributedLogManager dlm = createNewDLM(conf, name); BKAsyncLogWriter writer = (BKAsyncLogWriter)(dlm.startAsyncLogSegmentNonPartitioned()); writer.write(DLMTestUtil.getLogRecordInstance(1L)); writer.closeAndComplete(); Future<AsyncLogReader> futureReader1 = dlm.getAsyncLogReaderWithLock(DLSN.InitialDLSN); BKAsyncLogReaderDLSN reader1 = (BKAsyncLogReaderDLSN) Await.result(futureReader1); LogRecordWithDLSN record = Await.result(reader1.readNext()); assertEquals(1L, record.getTransactionId()); assertEquals(0L, record.getSequenceId()); DLMTestUtil.verifyLogRecord(record); String readLockPath = reader1.bkLedgerManager.getReadLockPath(); Utils.close(reader1); // simulate a old stream created without readlock path writer.bkDistributedLogManager.getWriterZKC().get().delete(readLockPath, -1); Future<AsyncLogReader> futureReader2 = dlm.getAsyncLogReaderWithLock(DLSN.InitialDLSN); AsyncLogReader reader2 = Await.result(futureReader2); record = Await.result(reader2.readNext()); assertEquals(1L, record.getTransactionId()); assertEquals(0L, record.getSequenceId()); DLMTestUtil.verifyLogRecord(record); }