private static LocalRecoverable deserializeV1(byte[] serialized) throws IOException { final ByteBuffer bb = ByteBuffer.wrap(serialized).order(ByteOrder.LITTLE_ENDIAN); if (bb.getInt() != MAGIC_NUMBER) { throw new IOException("Corrupt data: Unexpected magic number."); } final long offset = bb.getLong(); final byte[] targetFileBytes = new byte[bb.getInt()]; final byte[] tempFileBytes = new byte[bb.getInt()]; bb.get(targetFileBytes); bb.get(tempFileBytes); final String targetPath = new String(targetFileBytes, CHARSET); final String tempPath = new String(tempFileBytes, CHARSET); return new LocalRecoverable(new File(targetPath), new File(tempPath), offset); } }
@Override public byte[] serialize(LocalRecoverable obj) throws IOException { final byte[] targetFileBytes = obj.targetFile().getAbsolutePath().getBytes(CHARSET); final byte[] tempFileBytes = obj.tempFile().getAbsolutePath().getBytes(CHARSET); final byte[] targetBytes = new byte[20 + targetFileBytes.length + tempFileBytes.length]; ByteBuffer bb = ByteBuffer.wrap(targetBytes).order(ByteOrder.LITTLE_ENDIAN); bb.putInt(MAGIC_NUMBER); bb.putLong(obj.offset()); bb.putInt(targetFileBytes.length); bb.putInt(tempFileBytes.length); bb.put(targetFileBytes); bb.put(tempFileBytes); return targetBytes; }
LocalRecoverableFsDataOutputStream(LocalRecoverable resumable) throws IOException { this.targetFile = checkNotNull(resumable.targetFile()); this.tempFile = checkNotNull(resumable.tempFile()); if (!tempFile.exists()) { throw new FileNotFoundException("File Not Found: " + tempFile.getAbsolutePath()); } this.fileChannel = FileChannel.open(tempFile.toPath(), StandardOpenOption.WRITE, StandardOpenOption.APPEND); if (this.fileChannel.position() < resumable.offset()) { throw new IOException("Missing data in tmp file: " + tempFile.getAbsolutePath()); } this.fileChannel.truncate(resumable.offset()); this.fos = Channels.newOutputStream(fileChannel); }
@Override public Committer closeForCommit() throws IOException { final long pos = getPos(); close(); return new LocalCommitter(new LocalRecoverable(targetFile, tempFile, pos)); }
@Override public void commit() throws IOException { final File src = recoverable.tempFile(); final File dest = recoverable.targetFile(); // sanity check if (src.length() != recoverable.offset()) { // something was done to this file since the committer was created. // this is not the "clean" case throw new IOException("Cannot clean commit: File has trailing junk data."); } // rather than fall into default recovery, handle errors explicitly // in order to improve error messages try { Files.move(src.toPath(), dest.toPath(), StandardCopyOption.ATOMIC_MOVE); } catch (UnsupportedOperationException | AtomicMoveNotSupportedException e) { if (!src.renameTo(dest)) { throw new IOException("Committing file failed, could not rename " + src + " -> " + dest); } } catch (FileAlreadyExistsException e) { throw new IOException("Committing file failed. Target file already exists: " + dest); } }
@Override public ResumeRecoverable persist() throws IOException { // we call both flush and sync in order to ensure persistence on mounted // file systems, like NFS, EBS, EFS, ... flush(); sync(); return new LocalRecoverable(targetFile, tempFile, getPos()); }
@Override public void commitAfterRecovery() throws IOException { final File src = recoverable.tempFile(); final File dest = recoverable.targetFile(); final long expectedLength = recoverable.offset(); if (src.exists()) { if (src.length() > expectedLength) { // can happen if we co from persist to recovering for commit directly // truncate the trailing junk away try (FileOutputStream fos = new FileOutputStream(src, true)) { fos.getChannel().truncate(expectedLength); } } else if (src.length() < expectedLength) { throw new IOException("Missing data in tmp file: " + src); } // source still exists, so no renaming happened yet. do it! Files.move(src.toPath(), dest.toPath(), StandardCopyOption.ATOMIC_MOVE); } else if (!dest.exists()) { // neither exists - that can be a sign of // - (1) a serious problem (file system loss of data) // - (2) a recovery of a savepoint that is some time old and the users // removed the files in the meantime. // TODO how to handle this? // We probably need an option for users whether this should log, // or result in an exception or unrecoverable exception } }
private static LocalRecoverable deserializeV1(byte[] serialized) throws IOException { final ByteBuffer bb = ByteBuffer.wrap(serialized).order(ByteOrder.LITTLE_ENDIAN); if (bb.getInt() != MAGIC_NUMBER) { throw new IOException("Corrupt data: Unexpected magic number."); } final long offset = bb.getLong(); final byte[] targetFileBytes = new byte[bb.getInt()]; final byte[] tempFileBytes = new byte[bb.getInt()]; bb.get(targetFileBytes); bb.get(tempFileBytes); final String targetPath = new String(targetFileBytes, CHARSET); final String tempPath = new String(tempFileBytes, CHARSET); return new LocalRecoverable(new File(targetPath), new File(tempPath), offset); } }
@Override public byte[] serialize(LocalRecoverable obj) throws IOException { final byte[] targetFileBytes = obj.targetFile().getAbsolutePath().getBytes(CHARSET); final byte[] tempFileBytes = obj.tempFile().getAbsolutePath().getBytes(CHARSET); final byte[] targetBytes = new byte[20 + targetFileBytes.length + tempFileBytes.length]; ByteBuffer bb = ByteBuffer.wrap(targetBytes).order(ByteOrder.LITTLE_ENDIAN); bb.putInt(MAGIC_NUMBER); bb.putLong(obj.offset()); bb.putInt(targetFileBytes.length); bb.putInt(tempFileBytes.length); bb.put(targetFileBytes); bb.put(tempFileBytes); return targetBytes; }
@Override public Committer closeForCommit() throws IOException { final long pos = getPos(); close(); return new LocalCommitter(new LocalRecoverable(targetFile, tempFile, pos)); }
LocalRecoverableFsDataOutputStream(LocalRecoverable resumable) throws IOException { this.targetFile = checkNotNull(resumable.targetFile()); this.tempFile = checkNotNull(resumable.tempFile()); if (!tempFile.exists()) { throw new FileNotFoundException("File Not Found: " + tempFile.getAbsolutePath()); } this.fileChannel = FileChannel.open(tempFile.toPath(), StandardOpenOption.WRITE, StandardOpenOption.APPEND); if (this.fileChannel.position() < resumable.offset()) { throw new IOException("Missing data in tmp file: " + tempFile.getAbsolutePath()); } this.fileChannel.truncate(resumable.offset()); this.fos = Channels.newOutputStream(fileChannel); }
@Override public ResumeRecoverable persist() throws IOException { // we call both flush and sync in order to ensure persistence on mounted // file systems, like NFS, EBS, EFS, ... flush(); sync(); return new LocalRecoverable(targetFile, tempFile, getPos()); }
@Override public void commit() throws IOException { final File src = recoverable.tempFile(); final File dest = recoverable.targetFile(); // sanity check if (src.length() != recoverable.offset()) { // something was done to this file since the committer was created. // this is not the "clean" case throw new IOException("Cannot clean commit: File has trailing junk data."); } // rather than fall into default recovery, handle errors explicitly // in order to improve error messages try { Files.move(src.toPath(), dest.toPath(), StandardCopyOption.ATOMIC_MOVE); } catch (UnsupportedOperationException | AtomicMoveNotSupportedException e) { if (!src.renameTo(dest)) { throw new IOException("Committing file failed, could not rename " + src + " -> " + dest); } } catch (FileAlreadyExistsException e) { throw new IOException("Committing file failed. Target file already exists: " + dest); } }
@Override public void commitAfterRecovery() throws IOException { final File src = recoverable.tempFile(); final File dest = recoverable.targetFile(); final long expectedLength = recoverable.offset(); if (src.exists()) { if (src.length() > expectedLength) { // can happen if we co from persist to recovering for commit directly // truncate the trailing junk away try (FileOutputStream fos = new FileOutputStream(src, true)) { fos.getChannel().truncate(expectedLength); } } else if (src.length() < expectedLength) { throw new IOException("Missing data in tmp file: " + src); } // source still exists, so no renaming happened yet. do it! Files.move(src.toPath(), dest.toPath(), StandardCopyOption.ATOMIC_MOVE); } else if (!dest.exists()) { // neither exists - that can be a sign of // - (1) a serious problem (file system loss of data) // - (2) a recovery of a savepoint that is some time old and the users // removed the files in the meantime. // TODO how to handle this? // We probably need an option for users whether this should log, // or result in an exception or unrecoverable exception } }