/** * Create a processor that writes to a given file * @param outputName Name of file to write output to */ public BinaryEditsVisitor(String outputName) throws IOException { this.elfos = new EditLogFileOutputStream(new Configuration(), new File(outputName), 0); elfos.create(NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); }
/** * Finish the visitor */ @Override public void close(Throwable error) throws IOException { elfos.setReadyToFlush(); elfos.flushAndSync(true); elfos.close(); }
/** * Create empty edits logs file. */ @Override public void create(int layoutVersion) throws IOException { fc.truncate(0); fc.position(0); writeHeader(layoutVersion, doubleBuf.getCurrentBuf()); setReadyToFlush(); flush(); }
EditLogFileInputStream elfis = null; try { elfos = new EditLogFileOutputStream(new Configuration(), TEST_LOG_NAME, 0); elfos.create(NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); elfos.writeRaw(garbage, 0, garbage.length); elfos.setReadyToFlush(); elfos.flushAndSync(true); elfos.close(); elfos = null; elfis = new EditLogFileInputStream(TEST_LOG_NAME); if ((elfos != null) && (elfos.isOpen())) elfos.close(); if (elfis != null) elfis.close();
EditLogFileInputStream elfis = null; try { elfos = new EditLogFileOutputStream(new Configuration(), TEST_LOG_NAME, 0); elfos.create(NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); elfos.setReadyToFlush(); elfos.flushAndSync(true); elfos.close(); elfos = null; elfis = new EditLogFileInputStream(TEST_LOG_NAME);
EditLogFileOutputStream elos = new EditLogFileOutputStream(conf, TEST_EDITS, 0); try { byte[] small = new byte[] { 1, 2, 3, 4, 5, 8, 7 }; elos.create(NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); elos.writeRaw(small, 0, small.length); flushAndCheckLength(elos, MIN_PREALLOCATION_LENGTH); elos.writeRaw(small, 0, small.length); flushAndCheckLength(elos, MIN_PREALLOCATION_LENGTH); while (total > 0) { int toWrite = (total > buf.length) ? buf.length : total; elos.writeRaw(buf, 0, toWrite); total -= toWrite; } finally { if (elos != null) elos.close();
NNStorage.getInProgressEditsFileName(3)); EditLogFileOutputStream stream = new EditLogFileOutputStream(conf, log, 1024); try { stream.create(NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); if (!inBothDirs) { break; stream.close();
/** * Tests EditLogFileOutputStream doesn't throw NullPointerException on * close/abort sequence. See HDFS-2011. */ @Test public void testEditLogFileOutputStreamCloseAbort() throws IOException { // abort after a close should just ignore EditLogFileOutputStream editLogStream = new EditLogFileOutputStream(conf, TEST_EDITS, 0); editLogStream.close(); editLogStream.abort(); }
/** * Tests EditLogFileOutputStream doesn't throw NullPointerException on being * closed twice. */ public void testEditLogFileOutputStreamCloses() throws IOException, NullPointerException { // Testing EditLogFileOutputStream doesn't throw NullPointerException on // being closed twice File testEdits = new File(System.getProperty("test.build.data", "/tmp"), "editLogStream.dat"); try { EditLogFileOutputStream editLogStream = new EditLogFileOutputStream( testEdits, 0); editLogStream.close(); // Closing an twice should not throw a NullPointerException editLogStream.close(); } finally { // Cleanup the editLogStream.dat file we created testEdits.delete(); } // Successfully tested EditLogFileOutputStream doesn't throw // NullPointerException on being closed twice }
/** * Replace the journal at index <code>index</code> with one that throws an * exception on flush. * * @param index the index of the journal to take offline. * @return the original <code>EditLogOutputStream</code> of the journal. */ private void invalidateEditsDirAtIndex(int index, boolean failOnFlush, boolean failOnWrite) throws IOException { JournalAndStream jas = getJournalAndStream(index); EditLogFileOutputStream spyElos = spyOnStream(jas); if (failOnWrite) { doThrow(new IOException("fail on write()")).when(spyElos).write( (FSEditLogOp) any()); } if (failOnFlush) { doThrow(new IOException("fail on flush()")).when(spyElos).flush(); } else { doThrow(new IOException("fail on setReadyToFlush()")).when(spyElos) .setReadyToFlush(); } }
static void flushAndCheckLength(EditLogFileOutputStream elos, long expectedLength) throws IOException { elos.setReadyToFlush(); elos.flushAndSync(true); assertEquals(expectedLength, elos.getFile().length()); }
/** * Create empty edits logs file. */ @Override public void create() throws IOException { fc.truncate(0); fc.position(0); doubleBuf.getCurrentBuf().writeInt(FSConstants.LAYOUT_VERSION); setReadyToFlush(); flush(); }
@Override synchronized public EditLogOutputStream startLogSegment(long txid, int layoutVersion) throws IOException { try { currentInProgress = NNStorage.getInProgressEditsFile(sd, txid); EditLogOutputStream stm = new EditLogFileOutputStream(conf, currentInProgress, outputBufferCapacity); stm.create(layoutVersion); return stm; } catch (IOException e) { LOG.warn("Unable to start log segment " + txid + " at " + currentInProgress + ": " + e.getLocalizedMessage()); errorReporter.reportErrorOnFile(currentInProgress); throw e; } }
/** * If there is an IO Error on any log operations, remove that * directory from the list of directories. * If no more directories remain, then exit. */ synchronized void processIOError(int index) { if (editStreams == null || editStreams.size() <= 1) { FSNamesystem.LOG.fatal( "Fatal Error : All storage directories are inaccessible."); Runtime.getRuntime().exit(-1); } assert(index < getNumStorageDirs()); assert(getNumStorageDirs() == editStreams.size()); EditLogFileOutputStream eStream = (EditLogFileOutputStream)editStreams.get(index); File parentStorageDir = ((EditLogFileOutputStream)editStreams .get(index)).getFile() .getParentFile().getParentFile(); try { eStream.close(); } catch (Exception e) {} editStreams.remove(index); // // Invoke the ioerror routine of the fsimage // fsimage.processIOError(parentStorageDir); }
/** * Tests EditLogFileOutputStream doesn't throw NullPointerException on being * abort/abort sequence. See HDFS-2011. */ @Test public void testEditLogFileOutputStreamAbortAbort() throws IOException { // abort after a close should just ignore EditLogFileOutputStream editLogStream = null; try { editLogStream = new EditLogFileOutputStream(conf, TEST_EDITS, 0); editLogStream.abort(); editLogStream.abort(); } finally { IOUtils.cleanup(null, editLogStream); } } }
/** * Flush ready buffer to persistent store. currentBuffer is not flushed as it * accumulates new log records while readyBuffer will be flushed and synced. */ @Override public void flushAndSync(boolean durable) throws IOException { if (fp == null) { throw new IOException("Trying to use aborted output stream"); } if (doubleBuf.isFlushed()) { LOG.info("Nothing to flush"); return; } preallocate(); // preallocate file if necessary doubleBuf.flushTo(fp); if (durable && !shouldSkipFsyncForTests && !shouldSyncWritesAndSkipFsync) { fc.force(false); // metadata updates not needed } }
@Override public void visitOp(FSEditLogOp op) throws IOException { elfos.write(op); } }
doAnswer(blockingFlush).when(spyElos).flush();
Mockito.verify(nonRequiredSpy, Mockito.never()).setReadyToFlush(); assertFalse(nonRequiredJas.isActive());
private static void createEmptyInProgressEditLog(MiniDFSCluster cluster, NameNode nn, boolean writeHeader) throws IOException { long txid = nn.getNamesystem().getEditLog().getLastWrittenTxId(); URI sharedEditsUri = cluster.getSharedEditsDir(0, 1); File sharedEditsDir = new File(sharedEditsUri.getPath()); StorageDirectory storageDir = new StorageDirectory(sharedEditsDir); File inProgressFile = NameNodeAdapter.getInProgressEditsFile(storageDir, txid + 1); assertTrue("Failed to create in-progress edits file", inProgressFile.createNewFile()); if (writeHeader) { DataOutputStream out = new DataOutputStream(new FileOutputStream( inProgressFile)); EditLogFileOutputStream.writeHeader( NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION, out); out.close(); } }