@Override public synchronized void flush() throws IOException { if (mIsClosed || mDataOutputStream.size() == 0) { // There is nothing to flush. return; } try { mDataOutputStream.flush(); } catch (IOException e) { mRotateLogForNextWrite = true; throw new IOException(ExceptionMessage.JOURNAL_FLUSH_FAILURE.getMessageWithUrl( RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, mCurrentLog, e.getMessage()), e); } boolean overSize = mDataOutputStream.size() >= mMaxLogSize; if (overSize || !mUfs.supportsFlush()) { // (1) The log file is oversize, needs to be rotated. Or // (2) Underfs is S3 or OSS, flush on S3OutputStream/OSSOutputStream will only flush to // local temporary file, // call close and complete the log to sync the journal entry to S3/OSS. if (overSize) { LOG.info("Rotating log file. size: {} maxSize: {}", mDataOutputStream.size(), mMaxLogSize); } mRotateLogForNextWrite = true; } }
@Override public String createBlock(long sessionId, long blockId, String tierAlias, long initialBytes) throws BlockAlreadyExistsException, WorkerOutOfSpaceException, IOException { BlockStoreLocation loc = BlockStoreLocation.anyDirInTier(tierAlias); TempBlockMeta createdBlock; try { createdBlock = mBlockStore.createBlock(sessionId, blockId, loc, initialBytes); } catch (WorkerOutOfSpaceException e) { InetSocketAddress address = InetSocketAddress.createUnresolved(mAddress.getHost(), mAddress.getRpcPort()); throw new WorkerOutOfSpaceException(ExceptionMessage.CANNOT_REQUEST_SPACE .getMessageWithUrl(RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, address, blockId), e); } return createdBlock.getPath(); }
/** * The given entry should not have its sequence number set. This method will add the proper * sequence number to the passed in entry. * * @param entry an entry to write to the journal checkpoint file */ @Override public synchronized void write(JournalEntry entry) throws IOException { if (mIsClosed) { throw new IOException(ExceptionMessage.JOURNAL_WRITE_AFTER_CLOSE.getMessage()); } if (mRotateLogForNextWrite) { rotateLog(); mRotateLogForNextWrite = false; } try { mJournalFormatter.serialize( entry.toBuilder().setSequenceNumber(mJournalWriter.getNextSequenceNumber()).build(), mDataOutputStream); } catch (IOException e) { mRotateLogForNextWrite = true; throw new IOException(ExceptionMessage.JOURNAL_WRITE_FAILURE.getMessageWithUrl( RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, mCurrentLog, e.getMessage()), e); } }
JournalEntry firstEntryToFlush = mEntriesToFlush.peek(); if (firstEntryToFlush.getSequenceNumber() > lastPersistSeq + 1) { throw new RuntimeException(ExceptionMessage.JOURNAL_ENTRY_MISSING.getMessageWithUrl( RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, lastPersistSeq + 1, firstEntryToFlush.getSequenceNumber())); } catch (IOException e) { throw new IOException(ExceptionMessage.JOURNAL_WRITE_FAILURE .getMessageWithUrl(RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, mJournalOutputStream.currentLog(), e.getMessage()), e);
public synchronized void write(JournalEntry entry) throws IOException, JournalClosedException { try { maybeRecoverFromUfsFailures(); maybeRotateLog(); } catch (IOJournalClosedException e) { throw e.toJournalClosedException(); } try { JournalEntry entryToWrite = entry.toBuilder().setSequenceNumber(mNextSequenceNumber).build(); entryToWrite.writeDelimitedTo(mJournalOutputStream); LOG.debug("Adding journal entry (seq={}) to retryList with {} entries.", entryToWrite.getSequenceNumber(), mEntriesToFlush.size()); mEntriesToFlush.add(entryToWrite); mNextSequenceNumber++; } catch (IOJournalClosedException e) { throw e.toJournalClosedException(); } catch (IOException e) { // Set mNeedsRecovery to true so that {@code maybeRecoverFromUfsFailures} // can know a UFS failure has occurred. mNeedsRecovery = true; throw new IOException(ExceptionMessage.JOURNAL_WRITE_FAILURE .getMessageWithUrl(RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, mJournalOutputStream.currentLog(), e.getMessage()), e); } }
mJournalOutputStream = null; throw new IOException(ExceptionMessage.JOURNAL_FLUSH_FAILURE .getMessageWithUrl(RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, currentLog, e.getMessage()), e);
@Test public void missingIncompleteJournalFile() throws Exception { long startSN = 0x10; UfsJournalLogWriter writer = new UfsJournalLogWriter(mJournal, startSN); long nextSN = writeJournalEntries(writer, startSN, 5); DataOutputStream badOut = createMockDataOutputStream(writer); Mockito.doThrow(new IOException(INJECTED_IO_ERROR_MESSAGE)).when(badOut) .write(Mockito.any(byte[].class), Mockito.anyInt(), Mockito.anyInt()); tryWriteAndExpectToFail(writer, nextSN); UfsJournalSnapshot snapshot = UfsJournalSnapshot.getSnapshot(mJournal); UfsJournalFile journalFile = snapshot.getCurrentLog(mJournal); File file = new File(journalFile.getLocation().toString()); file.delete(); mThrown.expect(RuntimeException.class); mThrown.expectMessage( ExceptionMessage.JOURNAL_ENTRY_MISSING.getMessageWithUrl( RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, 0, 0x10)); writer.write(newEntry(nextSN)); writer.close(); }
ExceptionMessage.JOURNAL_ENTRY_MISSING.getMessageWithUrl( RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, firstCorruptedEntrySeq, seqOfFirstEntryToFlush));
@Override public synchronized void flush() throws IOException { if (mIsClosed || mDataOutputStream.size() == 0) { // There is nothing to flush. return; } try { mDataOutputStream.flush(); } catch (IOException e) { mRotateLogForNextWrite = true; throw new IOException(ExceptionMessage.JOURNAL_FLUSH_FAILURE.getMessageWithUrl( RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, mCurrentLog, e.getMessage()), e); } boolean overSize = mDataOutputStream.size() >= mMaxLogSize; if (overSize || !mUfs.supportsFlush()) { // (1) The log file is oversize, needs to be rotated. Or // (2) Underfs is S3 or OSS, flush on S3OutputStream/OSSOutputStream will only flush to // local temporary file, // call close and complete the log to sync the journal entry to S3/OSS. if (overSize) { LOG.info("Rotating log file. size: {} maxSize: {}", mDataOutputStream.size(), mMaxLogSize); } mRotateLogForNextWrite = true; } }
/** * The given entry should not have its sequence number set. This method will add the proper * sequence number to the passed in entry. * * @param entry an entry to write to the journal checkpoint file */ @Override public synchronized void write(JournalEntry entry) throws IOException { if (mIsClosed) { throw new IOException(ExceptionMessage.JOURNAL_WRITE_AFTER_CLOSE.getMessage()); } if (mRotateLogForNextWrite) { rotateLog(); mRotateLogForNextWrite = false; } try { mJournalFormatter.serialize( entry.toBuilder().setSequenceNumber(mJournalWriter.getNextSequenceNumber()).build(), mDataOutputStream); } catch (IOException e) { mRotateLogForNextWrite = true; throw new IOException(ExceptionMessage.JOURNAL_WRITE_FAILURE.getMessageWithUrl( RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, mCurrentLog, e.getMessage()), e); } }
JournalEntry firstEntryToFlush = mEntriesToFlush.peek(); if (firstEntryToFlush.getSequenceNumber() > lastPersistSeq + 1) { throw new RuntimeException(ExceptionMessage.JOURNAL_ENTRY_MISSING.getMessageWithUrl( RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, lastPersistSeq + 1, firstEntryToFlush.getSequenceNumber())); } catch (IOException e) { throw new IOException(ExceptionMessage.JOURNAL_WRITE_FAILURE .getMessageWithUrl(RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, mJournalOutputStream.currentLog(), e.getMessage()), e);
public synchronized void write(JournalEntry entry) throws IOException, JournalClosedException { try { maybeRecoverFromUfsFailures(); maybeRotateLog(); } catch (IOJournalClosedException e) { throw e.toJournalClosedException(); } try { JournalEntry entryToWrite = entry.toBuilder().setSequenceNumber(mNextSequenceNumber).build(); entryToWrite.writeDelimitedTo(mJournalOutputStream); LOG.debug("Adding journal entry (seq={}) to retryList with {} entries.", entryToWrite.getSequenceNumber(), mEntriesToFlush.size()); mEntriesToFlush.add(entryToWrite); mNextSequenceNumber++; } catch (IOJournalClosedException e) { throw e.toJournalClosedException(); } catch (IOException e) { // Set mNeedsRecovery to true so that {@code maybeRecoverFromUfsFailures} // can know a UFS failure has occurred. mNeedsRecovery = true; throw new IOException(ExceptionMessage.JOURNAL_WRITE_FAILURE .getMessageWithUrl(RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, mJournalOutputStream.currentLog(), e.getMessage()), e); } }
mJournalOutputStream = null; throw new IOException(ExceptionMessage.JOURNAL_FLUSH_FAILURE .getMessageWithUrl(RuntimeConstants.ALLUXIO_DEBUG_DOCS_URL, currentLog, e.getMessage()), e);