public static Reader of(LogRecordWithDLSN record) throws IOException { Preconditions.checkArgument(record.isRecordSet(), "record is not a recordset"); byte[] data = record.getPayload(); DLSN dlsn = record.getDlsn(); int startPosition = record.getPositionWithinLogSegment(); long startSequenceId = record.getStartSequenceIdOfCurrentSegment(); return new EnvelopedRecordSetReader( dlsn.getLogSegmentSequenceNo(), dlsn.getEntryId(), record.getTransactionId(), dlsn.getSlotId(), startPosition, startSequenceId, new ByteArrayInputStream(data)); }
private void dumpRecord(LogRecord record) { System.out.println("------------------------------------------------"); if (record instanceof LogRecordWithDLSN) { System.out.println("Record (txn = " + record.getTransactionId() + ", bytes = " + record.getPayload().length + ", dlsn = " + ((LogRecordWithDLSN) record).getDlsn() + ", sequence id = " + ((LogRecordWithDLSN) record).getSequenceId() + ")"); } else { System.out.println("Record (txn = " + record.getTransactionId() + ", bytes = " + record.getPayload().length + ")"); } System.out.println(""); if (skipPayload) { return; } if (printHex) { System.out.println(Hex.encodeHexString(record.getPayload())); } else { System.out.println(new String(record.getPayload(), UTF_8)); } }
nextRecordInStream = new LogRecordWithDLSN(recordStream.getCurrentPosition(), startSequenceId); nextRecordInStream.setMetadata(metadata); nextRecordInStream.setTransactionId(in.readLong()); nextRecordInStream.readPayload(in); if (LOG.isTraceEnabled()) { if (nextRecordInStream.isControl()) { LOG.trace("Reading {} Control DLSN {}", recordStream.getName(), nextRecordInStream.getDlsn()); } else { LOG.trace("Reading {} Valid DLSN {}", recordStream.getName(), nextRecordInStream.getDlsn()); if (!deserializeRecordSet && nextRecordInStream.isRecordSet()) { numRecords = LogRecordSet.numRecords(nextRecordInStream); if (deserializeRecordSet && nextRecordInStream.isRecordSet()) { recordSetReader = LogRecordSet.of(nextRecordInStream); } else {
@Override public String toString() { return "LogRecordWithDLSN{" + "dlsn=" + dlsn + ", txid=" + getTransactionId() + ", position=" + getPositionWithinLogSegment() + ", isControl=" + isControl() + ", isEndOfStream=" + isEndOfStream() + '}'; } }
private boolean recordPositionsContainsGap(LogRecordWithDLSN record, long lastPosition) { final boolean firstLogRecord = (1 == record.getPositionWithinLogSegment()); final boolean endOfStreamRecord = record.isEndOfStream(); final boolean emptyLogSegment = (0 == lastPosition); final boolean positionIncreasedByOne = (record.getPositionWithinLogSegment() == (lastPosition + 1)); return !firstLogRecord && !endOfStreamRecord && !emptyLogSegment && !positionIncreasedByOne; }
long getOffset() { return logRecord.getTransactionId() - logRecord.getPayload().length; } }
@Override public DLSN apply(LogRecordWithDLSN record) { return record.getDlsn(); } };
@Override public void onSuccess(LogRecordWithDLSN record) { System.out.println("Received record " + record.getDlsn()); System.out.println("\"\"\""); System.out.println(new String(record.getPayload(), UTF_8)); System.out.println("\"\"\""); lastDLSN.set(record.getDlsn()); reader.readNext().addEventListener(this); } };
@Override public void onSuccess(LogRecordWithDLSN record) { System.out.println("Received record " + record.getDlsn()); System.out.println("\"\"\""); System.out.println(new String(record.getPayload(), UTF_8)); System.out.println("\"\"\""); long diffInMilliseconds = System.currentTimeMillis() - record.getTransactionId(); if (!caughtup.get() && diffInMilliseconds < 2000) { System.out.println("Reader caught with latest data"); caughtup.set(true); } reader.readNext().addEventListener(this); } };
/** * Find the id of the last edit log transaction written to a edit log * ledger. */ protected Pair<Long, DLSN> readLastTxIdInLedger(LogSegmentMetadata l) throws IOException { LogRecordWithDLSN record = recoverLastRecordInLedger(l, false, false, true); if (null == record) { return Pair.of(DistributedLogConstants.EMPTY_LOGSEGMENT_TX_ID, DLSN.InvalidDLSN); } else { return Pair.of(record.getTransactionId(), record.getDlsn()); } }
LogRecordWithInputStream(LogRecordWithDLSN logRecord) { Preconditions.checkNotNull(logRecord); LOG.debug("Got record dlsn = {}, txid = {}, len = {}", new Object[] {logRecord.getDlsn(), logRecord.getTransactionId(), logRecord.getPayload().length}); this.logRecord = logRecord; this.payloadStream = logRecord.getPayLoadInputStream(); }
if (lastReadAheadDLSN.compareTo(record.getDlsn()) >= 0) { LOG.error("Out of order reads last {} : curr {}", lastReadAheadDLSN, record.getDlsn()); throw new LogReadException("Out of order reads"); lastReadAheadDLSN = record.getDlsn(); if (record.isControl()) { continue; if (minActiveDLSN.get().compareTo(record.getDlsn()) > 0) { continue; long deliveryMs = currentMs - record.getTransactionId(); if (deliveryMs >= 0) { readAheadDeliveryLatencyStat.registerSuccessfulEvent(deliveryMs); new Object[] { record.getDlsn(), streamName, record.getTransactionId(), currentMs, deliveryMs, reason }); cacheBytes.addAndGet(record.getPayload().length);
private static void transform(final AsyncLogWriter writer, LogRecordWithDLSN record, Transformer<byte[], byte[]> replicationTransformer, final CountDownLatch keepAliveLatch) throws Exception { DLSN srcDLSN = record.getDlsn(); byte[] payload = record.getPayload(); byte[] transformedPayload = replicationTransformer.transform(payload); TransformedRecord transformedRecord = new TransformedRecord(ByteBuffer.wrap(transformedPayload)); transformedRecord.setSrcDlsn(srcDLSN.serializeBytes()); ByteArrayOutputStream baos = new ByteArrayOutputStream(4096); transformedRecord.write(protocolFactory.getProtocol(new TIOStreamTransport(baos))); byte[] data = baos.toByteArray(); writer.write(new LogRecord(record.getSequenceId(), data)) .addEventListener(new FutureEventListener<DLSN>() { @Override public void onFailure(Throwable cause) { System.err.println("Encountered error on writing records to stream " + writer.getStreamName()); cause.printStackTrace(System.err); keepAliveLatch.countDown(); } @Override public void onSuccess(DLSN dlsn) { System.out.println("Write transformed record " + dlsn); } }); }
@Override public void process(LogRecordWithDLSN record) { if (found) { return; } this.result = record; if (record.getTransactionId() >= txId) { found = true; } }
if (null != record) { DLMTestUtil.verifyLogRecord(record); assertTrue(lastTxId < record.getTransactionId()); assertEquals(record.getTransactionId() - 1, record.getSequenceId()); lastTxId = record.getTransactionId(); numTrans++; continue;
continue; flags = lastRecordSkipTo.getMetadata(); currTxId = lastRecordSkipTo.getTransactionId(); LogRecordWithDLSN record = new LogRecordWithDLSN(recordStream.getCurrentPosition(), startSequenceId); record.setMetadata(flags); record.setTransactionId(currTxId); record.readPayload(in); recordSetReader = LogRecordSet.of(record); } else {
long expectedTxid = 0L; while (null != record) { Assert.assertEquals(expectedTxid, record.getTransactionId()); Assert.assertEquals(expectedTxid, record.getSequenceId()); Assert.assertEquals(new DLSN(1L, 0L, expectedTxid), record.getDlsn()); ++numReads; ++expectedTxid;
endTxId = lastRecord.getTransactionId(); recordCount = lastRecord.getLastPositionWithinLogSegment(); lastEntryId = lastRecord.getDlsn().getEntryId(); lastSlotId = lastRecord.getDlsn().getSlotId();
try { if (monotonic) { assertEquals(startSequenceId, value.getSequenceId()); } else { assertTrue(value.getSequenceId() < 0); assertTrue(value.getSequenceId() > startSequenceId); LOG.debug("Recevied record {} from {}", value.getDlsn(), reader.getStreamName()); assertTrue(!value.isControl()); assertTrue(value.getDlsn().getSlotId() == 0); assertTrue(value.getDlsn().compareTo(startPosition) >= 0); DLMTestUtil.verifyLargeLogRecord(value); } catch (Exception exc) { LOG.debug("Exception Encountered when verifying log record {} : ", value.getDlsn(), exc); errorsFound.set(true); completionLatch.countDown(); TestAsyncReaderWriter.readNext( reader, value.getDlsn().getNextDLSN(), monotonic ? value.getSequenceId() + 1 : value.getSequenceId(), monotonic, syncLatch,
} while (null != record && (record.isControl() || (record.getDlsn().compareTo(getStartDLSN()) < 0))); if (null == record) { break; } else { if (record.isEndOfStream() && !returnEndOfStreamRecord) { setLastException(new EndOfStreamException("End of Stream Reached for " + bkLedgerManager.getFullyQualifiedName())); bkDistributedLogManager.raiseAlert("Gap detected between records at dlsn = {}", record.getDlsn()); if (positionGapDetectionEnabled) { throw new DLIllegalStateException("Gap detected between records at dlsn = " + record.getDlsn()); lastPosition = record.getLastPositionWithinLogSegment(); + nextRequest.records.get(0).getDlsn()); nextRequest.setException(ise); if (null != request) { nextRequest.records.get(0).getDlsn()); setLastException(ise);