private static void parseJournalFile() { URI location; try { location = new URI(sJournalFile); } catch (URISyntaxException e) { throw new RuntimeException(e); } try (JournalFileParser parser = JournalFileParser.Factory.create(location)) { JournalEntry entry; while ((entry = parser.next()) != null) { if (entry.getSequenceNumber() < sStart) { continue; } if (entry.getSequenceNumber() >= sEnd) { break; } System.out.println(ENTRY_SEPARATOR); System.out.print(entry); } } catch (Exception e) { LOG.error("Failed to get next journal entry.", e); } }
@Override public JournalEntry read() throws IOException { int firstByte = inputStream.read(); if (firstByte == -1) { return null; } // All journal entries start with their size in bytes written as a varint. int size = ProtoUtils.readRawVarint32(firstByte, inputStream); byte[] buffer = size <= mBuffer.length ? mBuffer : new byte[size]; // Total bytes read so far for journal entry. int totalBytesRead = 0; while (totalBytesRead < size) { // Bytes read in last read request. int latestBytesRead = inputStream.read(buffer, totalBytesRead, size - totalBytesRead); if (latestBytesRead < 0) { break; } totalBytesRead += latestBytesRead; } if (totalBytesRead < size) { LOG.warn("Journal entry was truncated. Expected to read " + size + " bytes but only got " + totalBytesRead); return null; } JournalEntry entry = JournalEntry.parseFrom(new ByteArrayInputStream(buffer, 0, size)); if (entry != null) { mLatestSequenceNumber = entry.getSequenceNumber(); } return entry; }
while ((entry = parser.next()) != null) { if (start == -1) { start = entry.getSequenceNumber(); end = entry.getSequenceNumber();
result = result && (hasSequenceNumber() == other.hasSequenceNumber()); if (hasSequenceNumber()) { result = result && (getSequenceNumber() == other.getSequenceNumber());
hash = (37 * hash) + SEQUENCE_NUMBER_FIELD_NUMBER; hash = (53 * hash) + com.google.protobuf.Internal.hashLong( getSequenceNumber());
public Builder mergeFrom(alluxio.proto.journal.Journal.JournalEntry other) { if (other == alluxio.proto.journal.Journal.JournalEntry.getDefaultInstance()) return this; if (other.hasSequenceNumber()) { setSequenceNumber(other.getSequenceNumber());
JournalEntry entry; while ((entry = reader.read()) != null) { if (entry.getSequenceNumber() > lastPersistSeq) { lastPersistSeq = entry.getSequenceNumber();
if (!mEntriesToFlush.isEmpty()) { JournalEntry firstEntryToFlush = mEntriesToFlush.peek(); if (firstEntryToFlush.getSequenceNumber() > lastPersistSeq + 1) { throw new RuntimeException(ExceptionMessage.JOURNAL_ENTRY_MISSING.getMessageWithUrl( RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, lastPersistSeq + 1, firstEntryToFlush.getSequenceNumber())); if (entry.getSequenceNumber() > lastPersistSeq) { try { entry.toBuilder().build().writeDelimitedTo(mJournalOutputStream); retryEndSeq = entry.getSequenceNumber(); } catch (IOJournalClosedException e) { throw e.toJournalClosedException();
} else if (entry.getSequenceNumber() == mNextSequenceNumber) { mNextSequenceNumber++; return entry; } else if (entry.getSequenceNumber() < mNextSequenceNumber) { } else { throw new IllegalStateException(ExceptionMessage.JOURNAL_ENTRY_MISSING.getMessage( mNextSequenceNumber, entry.getSequenceNumber()));
/** * Applies the journal entry, ignoring empty entries and expanding multi-entries. * * @param entry the entry to apply */ private void applyEntry(JournalEntry entry) { Preconditions.checkState( entry.getAllFields().size() <= 1 || (entry.getAllFields().size() == 2 && entry.hasSequenceNumber()), "Raft journal entries should never set multiple fields in addition to sequence " + "number, but found %s", entry); if (entry.getJournalEntriesCount() > 0) { // This entry aggregates multiple entries. for (JournalEntry e : entry.getJournalEntriesList()) { applyEntry(e); } } else if (entry.getSequenceNumber() < 0) { // Negative sequence numbers indicate special entries used to indicate that a new primary is // starting to serve. mLastPrimaryStartSequenceNumber = entry.getSequenceNumber(); } else if (entry.toBuilder().clearSequenceNumber().build() .equals(JournalEntry.getDefaultInstance())) { // Ignore empty entries, they are created during snapshotting. } else { applySingleEntry(entry); } }
@Override public synchronized void install(SnapshotReader snapshotReader) { if (mClosed) { return; } if (mIgnoreApplys) { LOG.warn("Unexpected request to install a snapshot on a read-only journal state machine"); return; } resetState(); JournalEntryStreamReader reader = new JournalEntryStreamReader(new SnapshotReaderStream(snapshotReader)); JournalEntry entry = null; while (snapshotReader.hasRemaining()) { try { entry = reader.readEntry(); } catch (IOException e) { ProcessUtils.fatalError(LOG, e, "Failed to install snapshot"); } applyToMaster(entry); } long snapshotSN = entry != null ? entry.getSequenceNumber() : -1; if (snapshotSN < mNextSequenceNumberToRead - 1) { LOG.warn("Installed snapshot for SN {} but next SN to read is {}", snapshotSN, mNextSequenceNumberToRead); } mNextSequenceNumberToRead = snapshotSN + 1; LOG.info("Successfully installed snapshot up to SN {}", snapshotSN); }
public synchronized void write(JournalEntry entry) throws IOException, JournalClosedException { try { maybeRecoverFromUfsFailures(); maybeRotateLog(); } catch (IOJournalClosedException e) { throw e.toJournalClosedException(); } try { JournalEntry entryToWrite = entry.toBuilder().setSequenceNumber(mNextSequenceNumber).build(); entryToWrite.writeDelimitedTo(mJournalOutputStream); LOG.debug("Adding journal entry (seq={}) to retryList with {} entries.", entryToWrite.getSequenceNumber(), mEntriesToFlush.size()); mEntriesToFlush.add(entryToWrite); mNextSequenceNumber++; } catch (IOJournalClosedException e) { throw e.toJournalClosedException(); } catch (IOException e) { // Set mNeedsRecovery to true so that {@code maybeRecoverFromUfsFailures} // can know a UFS failure has occurred. mNeedsRecovery = true; throw new IOException(ExceptionMessage.JOURNAL_WRITE_FAILURE .getMessageWithUrl(RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, mJournalOutputStream.currentLog(), e.getMessage()), e); } }
/** * Reads logs created while reading the journal. */ @Test public void readNewLogs() throws Exception { long endSN = 10; buildCompletedLog(0, endSN); try (JournalReader reader = mJournal.getReader(true)) { Journal.JournalEntry entry; int sn = 0; while ((entry = reader.read()) != null) { Assert.assertEquals(sn, entry.getSequenceNumber()); sn++; } Assert.assertEquals(endSN, sn); Assert.assertEquals(sn, reader.getNextSequenceNumber()); // Write another two logs. buildCompletedLog(endSN, endSN * 2); buildIncompleteLog(endSN * 2, endSN * 2 + 1); while ((entry = reader.read()) != null) { Assert.assertEquals(sn, entry.getSequenceNumber()); sn++; } Assert.assertEquals(endSN * 2 + 1, sn); Assert.assertEquals(sn, reader.getNextSequenceNumber()); } }
@SuppressFBWarnings(value = "VO_VOLATILE_INCREMENT", justification = "All writes to mNextSequenceNumberToRead are synchronized") private synchronized void applySingleEntry(JournalEntry entry) { if (mClosed) { return; } long newSN = entry.getSequenceNumber(); if (newSN < mNextSequenceNumberToRead) { LOG.info("Ignoring duplicate journal entry with SN {} when next SN is {}", newSN, mNextSequenceNumberToRead); return; } if (newSN > mNextSequenceNumberToRead) { ProcessUtils.fatalError(LOG, "Unexpected journal entry. The next expected SN is {}, but" + " encountered an entry with SN {}. Full journal entry: {}", mNextSequenceNumberToRead, newSN, entry); } mNextSequenceNumberToRead++; if (!mIgnoreApplys) { applyToMaster(entry); } }
/** * Reads completed logs. */ @Test public void readCompletedLog() throws Exception { long fileSize = 10; long endSN = 10 * fileSize; for (long i = 0; i < endSN / fileSize; i++) { buildCompletedLog(i * fileSize, i * fileSize + fileSize); } try (JournalReader reader = mJournal.getReader(true)) { Journal.JournalEntry entry; int sn = 0; while ((entry = reader.read()) != null) { Assert.assertEquals(sn, entry.getSequenceNumber()); sn++; } Assert.assertEquals(endSN, sn); Assert.assertEquals(sn, reader.getNextSequenceNumber()); // Further reads should return null. Assert.assertTrue(reader.read() == null); } }
/** * Reads incomplete logs in a primary master. */ @Test public void readIncompleteLogPrimary() throws Exception { long endSN = 10; buildCompletedLog(0, endSN); buildIncompleteLog(endSN, endSN + 1); try (JournalReader reader = mJournal.getReader(true)) { Journal.JournalEntry entry; int sn = 0; while ((entry = reader.read()) != null) { Assert.assertEquals(sn, entry.getSequenceNumber()); sn++; } Assert.assertEquals(endSN + 1, sn); Assert.assertEquals(sn, reader.getNextSequenceNumber()); } }
/** * Checks that journal entries with sequence number between startSN (inclusive) and endSN * (exclusive) exist in the current journal files. * * @param startSN start sequence number (inclusive) * @param endSN end sequence number (exclusive) */ private void checkJournalEntries(long startSN, long endSN) throws IOException, InvalidJournalEntryException { try (JournalReader reader = new UfsJournalReader(mJournal, startSN, true)) { Journal.JournalEntry entry; long seq = startSN; while ((entry = reader.read()) != null) { Assert.assertEquals(seq, entry.getSequenceNumber()); seq++; } Assert.assertEquals(endSN, seq); } }
/** * Secondary master cannot read incomplete logs. */ @Test public void readIncompleteLogSecondary() throws Exception { long endSN = 10; buildCompletedLog(0, endSN); buildIncompleteLog(endSN, endSN + 1); try (JournalReader reader = mJournal.getReader(false)) { Journal.JournalEntry entry; int sn = 0; while ((entry = reader.read()) != null) { Assert.assertEquals(sn, entry.getSequenceNumber()); sn++; } Assert.assertEquals(endSN, sn); Assert.assertEquals(sn, reader.getNextSequenceNumber()); } }
/** * Reads checkpoint. */ @Test public void readCheckpoint() throws Exception { long endSN = CHECKPOINT_SIZE; buildCheckpoint(endSN); try (JournalReader reader = mJournal.getReader(true)) { Journal.JournalEntry entry; int sn = 0; while ((entry = reader.read()) != null) { Assert.assertEquals(sn, entry.getSequenceNumber()); sn++; } Assert.assertEquals(CHECKPOINT_SIZE, sn); Assert.assertEquals(endSN, reader.getNextSequenceNumber()); } }
private static void readFromJournal() { UfsJournal journal = new UfsJournalSystem(getJournalLocation(), 0).createJournal(new NoopMaster(sMaster)); try (JournalReader reader = new UfsJournalReader(journal, sStart, true)) { JournalEntry entry; while ((entry = reader.read()) != null) { if (entry.getSequenceNumber() >= sEnd) { break; } System.out.println(ENTRY_SEPARATOR); System.out.print(entry); } } catch (Exception e) { LOG.error("Failed to read next journal entry.", e); } }