RecoverableWriter.CommitRecoverable closeForCommit() throws IOException { return currentPartStream.closeForCommit().getRecoverable(); }
@Test public void testCloseWithNoData() throws Exception { final RecoverableWriter writer = getNewFileSystemWriter(); final Path testDir = getBasePathForTest(); final Path path = new Path(testDir, "part-0"); final RecoverableFsDataOutputStream stream = writer.open(path); for (Map.Entry<Path, String> fileContents : getFileContentByPath(testDir).entrySet()) { Assert.assertTrue(fileContents.getKey().getName().startsWith(".part-0.inprogress.")); Assert.assertTrue(fileContents.getValue().isEmpty()); } stream.closeForCommit().commit(); for (Map.Entry<Path, String> fileContents : getFileContentByPath(testDir).entrySet()) { Assert.assertEquals("part-0", fileContents.getKey().getName()); Assert.assertTrue(fileContents.getValue().isEmpty()); } }
@Test public void testCommitAfterNormalClose() throws Exception { final RecoverableWriter writer = getNewFileSystemWriter(); final Path testDir = getBasePathForTest(); final Path path = new Path(testDir, "part-0"); try (final RecoverableFsDataOutputStream stream = writer.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); stream.closeForCommit().commit(); for (Map.Entry<Path, String> fileContents : getFileContentByPath(testDir).entrySet()) { Assert.assertEquals("part-0", fileContents.getKey().getName()); Assert.assertEquals(testData1, fileContents.getValue()); } } }
@Test(expected = IOException.class) public void testExceptionWritingAfterCloseForCommit() throws Exception { final Path testDir = getBasePathForTest(); final RecoverableWriter writer = getNewFileSystemWriter(); final Path path = new Path(testDir, "part-0"); try (final RecoverableFsDataOutputStream stream = writer.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); stream.closeForCommit().getRecoverable(); stream.write(testData2.getBytes(StandardCharsets.UTF_8)); fail(); } }
@Test public void testCommitAfterPersist() throws Exception { final RecoverableWriter writer = getNewFileSystemWriter(); final Path testDir = getBasePathForTest(); final Path path = new Path(testDir, "part-0"); try (final RecoverableFsDataOutputStream stream = writer.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); stream.persist(); stream.write(testData2.getBytes(StandardCharsets.UTF_8)); stream.closeForCommit().commit(); for (Map.Entry<Path, String> fileContents : getFileContentByPath(testDir).entrySet()) { Assert.assertEquals("part-0", fileContents.getKey().getName()); Assert.assertEquals(testData1 + testData2, fileContents.getValue()); } } }
recoveredStream.closeForCommit().commit();
@Test(expected = IOException.class) public void testResumeAfterCommit() throws Exception { final Path testDir = getBasePathForTest(); final RecoverableWriter writer = getNewFileSystemWriter(); final Path path = new Path(testDir, "part-0"); RecoverableWriter.ResumeRecoverable recoverable; try (final RecoverableFsDataOutputStream stream = writer.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); recoverable = stream.persist(); stream.write(testData2.getBytes(StandardCharsets.UTF_8)); stream.closeForCommit().commit(); } // this should throw an exception as the file is already committed writer.recover(recoverable); fail(); }
recoverables.add(stream.closeForCommit().getRecoverable());
recoverables.add(stream.closeForCommit().getRecoverable());
@Test public void testCommitAfterRecovery() throws Exception { final Path testDir = getBasePathForTest(); final Path path = new Path(testDir, "part-0"); final RecoverableWriter initWriter = getNewFileSystemWriter(); final RecoverableWriter.CommitRecoverable recoverable; try (final RecoverableFsDataOutputStream stream = initWriter.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); stream.persist(); stream.persist(); // and write some more data stream.write(testData2.getBytes(StandardCharsets.UTF_8)); recoverable = stream.closeForCommit().getRecoverable(); } final byte[] serializedRecoverable = initWriter.getCommitRecoverableSerializer().serialize(recoverable); // get a new serializer from a new writer to make sure that no pre-initialized state leaks in. final RecoverableWriter newWriter = getNewFileSystemWriter(); final SimpleVersionedSerializer<RecoverableWriter.CommitRecoverable> deserializer = newWriter.getCommitRecoverableSerializer(); final RecoverableWriter.CommitRecoverable recoveredRecoverable = deserializer.deserialize(deserializer.getVersion(), serializedRecoverable); final RecoverableFsDataOutputStream.Committer committer = newWriter.recoverForCommit(recoveredRecoverable); committer.commitAfterRecovery(); Map<Path, String> files = getFileContentByPath(testDir); Assert.assertEquals(1L, files.size()); for (Map.Entry<Path, String> fileContents : files.entrySet()) { Assert.assertEquals("part-0", fileContents.getKey().getName()); Assert.assertEquals(testData1 + testData2, fileContents.getValue()); } }
RecoverableWriter.CommitRecoverable closeForCommit() throws IOException { return currentPartStream.closeForCommit().getRecoverable(); }
RecoverableWriter.CommitRecoverable closeForCommit() throws IOException { return currentPartStream.closeForCommit().getRecoverable(); }