Mutator(LogSegmentMetadata original) { super(original.getZkPath(), original.getVersion(), original.getLedgerId(), original.getFirstTxId()); this.inprogress = original.isInProgress(); this.logSegmentSequenceNo = original.getLogSegmentSequenceNumber(); this.lastEntryId = original.getLastEntryId(); this.lastSlotId = original.getLastSlotId(); this.lastTxId = original.getLastTxId(); this.completionTime = original.getCompletionTime(); this.recordCount = original.getRecordCount(); this.regionId = original.getRegionId(); this.status = original.getStatus(); this.minActiveEntryId = original.getMinActiveDLSN().getEntryId(); this.minActiveSlotId = original.getMinActiveDLSN().getSlotId(); this.startSequenceId = original.getStartSequenceId(); this.envelopeEntries = original.getEnvelopeEntries(); }
@Override public void updateLogSegment(Transaction<Object> txn, LogSegmentMetadata segment) { byte[] finalisedData = segment.getFinalisedData().getBytes(UTF_8); Op updateOp = Op.setData(segment.getZkPath(), finalisedData, -1); txn.addOp(DefaultZKOp.of(updateOp)); }
public static LogSegmentMetadata parseData(String path, byte[] data) throws IOException { return parseData(path, data, false); }
public long getStartSequenceId() { // generate negative sequence id for log segments that created <= v4 return supportsSequenceId() && startSequenceId != DistributedLogConstants.UNASSIGNED_SEQUENCE_ID ? startSequenceId : Long.MIN_VALUE + (getLogSegmentSequenceNumber() << 32L); }
public String toString() { return "[LedgerId:" + ledgerId + ", firstTxId:" + firstTxId + ", lastTxId:" + lastTxId + ", version:" + version + ", completionTime:" + completionTime + ", recordCount:" + recordCount + ", regionId:" + regionId + ", status:" + status + ", logSegmentSequenceNumber:" + getLogSegmentSequenceNumber() + ", lastEntryId:" + getLastEntryId() + ", lastSlotId:" + getLastSlotId() + ", inprogress:" + inprogress + ", minActiveDLSN:" + minActiveDLSN + ", startSequenceId:" + startSequenceId + "]"; }
completedLogSegment.write(zkc); inprogressLogSegment.write(zkc); assertEquals(goodLastDLSN1, updatedCompletedLogSegment.getLastDLSN()); assertEquals(goodRecord1.getTransactionId(), updatedCompletedLogSegment.getLastTxId()); assertTrue(updatedCompletedLogSegment.isRecordLastPositioninThisSegment(goodRecord1)); assertEquals(goodLastDLSN1, updatedCompletedLogSegment.getLastDLSN()); assertEquals(goodRecord1.getTransactionId(), updatedCompletedLogSegment.getLastTxId()); assertTrue(updatedCompletedLogSegment.isRecordLastPositioninThisSegment(goodRecord1)); assertEquals(goodLastDLSN1, readCompletedLogSegment.getLastDLSN()); assertEquals(goodRecord1.getTransactionId(), readCompletedLogSegment.getLastTxId()); assertTrue(readCompletedLogSegment.isRecordLastPositioninThisSegment(goodRecord1)); assertEquals(updatedCompletedLogSegment, readCompletedLogSegment); assertEquals(completedLogSegment.getCompletionTime(), readCompletedLogSegment.getCompletionTime()); assertEquals(completedLogSegment.getFirstTxId(), readCompletedLogSegment.getFirstTxId()); assertEquals(completedLogSegment.getLedgerId(), readCompletedLogSegment.getLedgerId()); assertEquals(completedLogSegment.getLogSegmentSequenceNumber(), readCompletedLogSegment.getLogSegmentSequenceNumber()); assertEquals(completedLogSegment.getRegionId(), readCompletedLogSegment.getRegionId()); assertEquals(completedLogSegment.getZkPath(), readCompletedLogSegment.getZkPath()); assertEquals(completedLogSegment.getZNodeName(), readCompletedLogSegment.getZNodeName());
private static void assertEqualsWithoutSequenceId(LogSegmentMetadata m1, LogSegmentMetadata m2) { assertEquals("expected " + m1 + " but got " + m2, m1.getLogSegmentSequenceNumber(), m2.getLogSegmentSequenceNumber()); assertEquals("expected " + m1 + " but got " + m2, m1.getLedgerId(), m2.getLedgerId()); assertEquals("expected " + m1 + " but got " + m2, m1.getFirstTxId(), m2.getFirstTxId()); assertEquals("expected " + m1 + " but got " + m2, m1.getLastTxId(), m2.getLastTxId()); assertEquals("expected " + m1 + " but got " + m2, m1.getLastDLSN(), m2.getLastDLSN()); assertEquals("expected " + m1 + " but got " + m2, m1.getRecordCount(), m2.getRecordCount()); assertEquals("expected " + m1 + " but got " + m2, m1.isInProgress(), m2.isInProgress()); }
&& prevSegment.getVersion() >= LogSegmentMetadata.LogSegmentMetadataVersion.VERSION_V2_LEDGER_SEQNO.value && segment.getVersion() >= LogSegmentMetadata.LogSegmentMetadataVersion.VERSION_V2_LEDGER_SEQNO.value && prevSegment.getLogSegmentSequenceNumber() != segment.getLogSegmentSequenceNumber() && prevSegment.getLogSegmentSequenceNumber() + 1 != segment.getLogSegmentSequenceNumber()) { LOG.error("{} found ledger sequence number gap between log segment {} and {}", new Object[] { streamName, prevSegment, segment }); throw new UnexpectedException(streamName + " found ledger sequence number gap between log segment " + prevSegment.getLogSegmentSequenceNumber() + " and " + segment.getLogSegmentSequenceNumber()); if (!segment.isInProgress()) { if (segment.supportsSequenceId()) { startSequenceId = segment.getStartSequenceId() + segment.getRecordCount(); if (null != prevSegment && prevSegment.supportsSequenceId() && prevSegment.getStartSequenceId() > segment.getStartSequenceId()) { LOG.warn("{} found decreasing start sequence id in log segment {}, previous is {}", new Object[] { streamName, segment, prevSegment }); if (segment.supportsSequenceId()) { LogSegmentMetadata newSegment = segment.mutator() .setStartSequenceId(startSequenceId == DistributedLogConstants.UNASSIGNED_SEQUENCE_ID ? 0L : startSequenceId) .build();
@Test(timeout = 60000) public void testReadMetadataCrossVersion() throws Exception { LogSegmentMetadata metadata1 = new LogSegmentMetadataBuilder("/metadata2", 1, 1000, 1).setRegionId(TEST_REGION_ID).build(); metadata1.write(zkc); // synchronous read LogSegmentMetadata read1 = FutureUtils.result(LogSegmentMetadata.read(zkc, "/metadata2", true)); assertEquals(read1.getLedgerId(), metadata1.getLedgerId()); assertEquals(read1.getFirstTxId(), metadata1.getFirstTxId()); assertEquals(read1.getLastTxId(), metadata1.getLastTxId()); assertEquals(read1.getLogSegmentSequenceNumber(), metadata1.getLogSegmentSequenceNumber()); assertEquals(DistributedLogConstants.LOCAL_REGION_ID, read1.getRegionId()); }
if (l.isTruncated() && isInitialPositioning && !conf.getIgnoreTruncationStatus()) { final boolean hasDataToRead = (l.getLastDLSN().compareTo(nextReadDLSN) >= 0); ((l.isInProgress() || (null != currentMetadata && currentMetadata.isInProgress())) && l.getLogSegmentSequenceNumber() == nextReadAheadPosition.getLogSegmentSequenceNumber()); if (l.isPartiallyTruncated() && !isInitialPositioning && (l.getMinActiveDLSN().compareTo(nextReadDLSN) > 0)) { if (conf.getAlertWhenPositioningOnTruncated()) { alertStatsLogger.raise("Trying to position reader on {} when {} is marked partially truncated", if (l.isPartiallyTruncated() && !conf.getIgnoreTruncationStatus()) { startBKEntry = l.getMinActiveDLSN().getEntryId(); readAheadCache.setMinActiveDLSN(l.getMinActiveDLSN()); if(l.getLogSegmentSequenceNumber() == nextReadAheadPosition.getLogSegmentSequenceNumber()) { startBKEntry = Math.max(startBKEntry, nextReadAheadPosition.getEntryId()); if (currentMetadata != null) { inProgressChanged = currentMetadata.isInProgress() && !l.isInProgress(); nextReadAheadPosition = new LedgerReadPosition(l.getLedgerId(), l.getLogSegmentSequenceNumber(), startBKEntry); if (conf.getTraceReadAheadMetadataChanges()) { LOG.info("Moved read position to {} for stream {} at {}.", if (l.isTruncated()) {
protected void repairLogSegment(BookKeeperAdmin bkAdmin, MetadataUpdater metadataUpdater, LogSegmentMetadata segment) throws Exception { if (segment.isInProgress()) { System.out.println("Skip inprogress log segment " + segment); return; LedgerHandle lh = bkAdmin.openLedger(segment.getLedgerId(), true); long lac = lh.getLastAddConfirmed(); Enumeration<LedgerEntry> entries = lh.readEntries(lac, lac); .setLogSegmentInfo(segment.getLogSegmentSequenceNumber(), segment.getStartSequenceId()) .setEntryId(lastEntry.getEntryId()) .setEnvelopeEntry(LogSegmentMetadata.supportsEnvelopedEntries(segment.getVersion())) .setInputStream(lastEntry.getEntryInputStream()) .buildReader();
/** * Compute the start sequence id for <code>segment</code>, based on previous segment list * <code>segmentListDesc</code>. * * @param logSegmentDescList * list of segments in descending order * @param segment * segment to compute start sequence id for * @return start sequence id */ public static long computeStartSequenceId(List<LogSegmentMetadata> logSegmentDescList, LogSegmentMetadata segment) throws UnexpectedException { long startSequenceId = 0L; for (LogSegmentMetadata metadata : logSegmentDescList) { if (metadata.getLogSegmentSequenceNumber() >= segment.getLogSegmentSequenceNumber()) { continue; } else if (metadata.getLogSegmentSequenceNumber() < (segment.getLogSegmentSequenceNumber() - 1)) { break; } if (metadata.isInProgress()) { throw new UnexpectedException("Should not complete log segment " + segment.getLogSegmentSequenceNumber() + " since it's previous log segment is still inprogress : " + logSegmentDescList); } if (metadata.supportsSequenceId()) { startSequenceId = metadata.getStartSequenceId() + metadata.getRecordCount(); } } return startSequenceId; }
+ l.toString() + " at path " + l.getZkPath() + ". Unable to continue recovery."); return Future.exception(new IOException("Unrecoverable corruption," } else if (endTxId == DistributedLogConstants.EMPTY_LOGSEGMENT_TX_ID) { endTxId = l.getFirstTxId(); l.getZNodeName(), l.getLogSegmentSequenceNumber(), l.getLedgerId(), l.getFirstTxId(), endTxId, recordCount,
LogSegmentMetadata segment = DLMTestUtil.completedLogSegment(ledgerPath, i, (i - 1) * 100, i * 100 - 1, 100, i, 100, 0); completedLogSegments.put(((long)i), segment); LOG.info("Create completed segment {} : {}", segment.getZkPath(), segment); segment.write(zkc); LOG.info("Create inprogress segment {} : {}", segment.getZkPath(), segment); segment.write(zkc); assertEquals(6L, segmentChanged.getLogSegmentSequenceNumber()); assertTrue(segmentChanged.isInProgress()); assertEquals(5 * 100, segmentChanged.getFirstTxId()); assertEquals(3L, segmentChanged.getLedgerId());
if (currentMetadata.isInProgress()) { // we don't want to fence the current journal if (null == currentLH) { if (conf.getTraceReadAheadMetadataChanges()) { if (lastAddConfirmed < currentMetadata.getLastEntryId()) { alertStatsLogger.raise("Unexpected last entry id during read ahead; {} , {}", currentMetadata, lastAddConfirmed); if (conf.getTraceReadAheadMetadataChanges() && lastAddConfirmed > (currentMetadata.getLastEntryId() + 1)) { LOG.warn("Potential Metadata Corruption {} for stream {}, lastAddConfirmed {}", new Object[] {currentMetadata, logMetadata.getFullyQualifiedName(), lastAddConfirmed}); if (currentMetadataIndex + 1 < ledgerList.size()) { currentMetadata = ledgerList.get(++currentMetadataIndex); if (currentMetadata.getLogSegmentSequenceNumber() != (oldMetadata.getLogSegmentSequenceNumber() + 1)) { setReadAheadError(tracker); } else { if (currentMetadata.isTruncated()) { if (conf.getAlertWhenPositioningOnTruncated()) { alertStatsLogger.raise("Trying to position reader on the log segment that is marked truncated : {}", currentMetadata, fullyQualifiedName); nextReadAheadPosition.positionOnNewLogSegment(currentMetadata.getLedgerId(), currentMetadata.getLogSegmentSequenceNumber());
if (inprogressLogSegment.getLedgerId() != ledgerId) { FutureUtils.setException(promise, new IOException( "Active ledger has different ID to inprogress. " + inprogressLogSegment.getLedgerId() + " found, " + ledgerId + " expected")); return; if (inprogressLogSegment.getFirstTxId() != firstTxId) { FutureUtils.setException(promise, new IOException("Transaction id not as expected, " + inprogressLogSegment.getFirstTxId() + " found, " + firstTxId + " expected")); return; if ((inprogressLogSegment.getLogSegmentSequenceNumber() != logSegmentSeqNo) || (leastInprogressLSSN != logSegmentSeqNo)) { FutureUtils.setException(promise, new UnexpectedException( inprogressLogSegment.completeLogSegment( pathForCompletedLedger, lastTxId, lastSlotId, startSequenceId); setLastLedgerRollingTimeMillis(completedLogSegment.getCompletionTime());
public long getLedgerId() { return logSegmentMetadata.getLedgerId(); }
Set<Long> inprogressSeqNos = new HashSet<Long>(); for (LogSegmentMetadata segment : segments) { if (segment.isInProgress()) { inprogressSeqNos.add(segment.getLogSegmentSequenceNumber()); if (!segment.isInProgress() && inprogressSeqNos.contains(segment.getLogSegmentSequenceNumber())) { isCandidate = true; long lastSeqNo = firstSegment.getLogSegmentSequenceNumber(); if (lastSeqNo + 1 != nextSegment.getLogSegmentSequenceNumber()) { isCandidate = true; break; LogSegmentMetadata segment = seg; List<String> dumpedEntries = new ArrayList<String>(); if (segment.isInProgress()) { LedgerHandle lh = bkc.get().openLedgerNoRecovery(segment.getLedgerId(), BookKeeper.DigestType.CRC32, dlConf.getBKDigestPW().getBytes(UTF_8)); try { long lac = lh.readLastConfirmed(); segment = segment.mutator().setLastEntryId(lac).build(); if (printInprogressOnly && dumpEntries && lac >= 0) { Enumeration<LedgerEntry> entries = lh.readEntries(0L, lac); if (segment.isInProgress()) { ledgers.add(Pair.of(segment, dumpedEntries));
public int compare(LogSegmentMetadata o1, LogSegmentMetadata o2) { if ((o1.getLogSegmentSequenceNumber() == DistributedLogConstants.UNASSIGNED_LOGSEGMENT_SEQNO) || (o2.getLogSegmentSequenceNumber() == DistributedLogConstants.UNASSIGNED_LOGSEGMENT_SEQNO)) { if (o1.firstTxId < o2.firstTxId) { return -1; } else if (o1.firstTxId == o2.firstTxId) { return 0; } else { return 1; } } else { if (o1.getLogSegmentSequenceNumber() < o2.getLogSegmentSequenceNumber()) { return -1; } else if (o1.getLogSegmentSequenceNumber() == o2.getLogSegmentSequenceNumber()) { // make sure we won't move over inprogress log segment if it still presents in the list if (o1.isInProgress() && !o2.isInProgress()) { return -1; } else if (!o1.isInProgress() && o2.isInProgress()) { return 1; } else { return 0; } } else { return 1; } } } };
LedgerEntry entry = entries.nextElement(); reader = Entry.newBuilder() .setLogSegmentInfo(metadata.getLogSegmentSequenceNumber(), metadata.getStartSequenceId()) .setEntryId(entry.getEntryId()) .setEnvelopeEntry( LogSegmentMetadata.supportsEnvelopedEntries(metadata.getVersion())) .deserializeRecordSet(true) .setInputStream(entry.getEntryInputStream())