/** Discovers all shards of this file. */ public List<String> readFilesWithRetries() throws IOException, InterruptedException { return readFilesWithRetries(Sleeper.DEFAULT, BACK_OFF_FACTORY.backoff()); }
@Test public void testReadEmpty() throws Exception { File emptyFile = tmpFolder.newFile("result-000-of-001"); Files.write("", emptyFile, StandardCharsets.UTF_8); FilePatternMatchingShardedFile shardedFile = new FilePatternMatchingShardedFile(filePattern); assertThat(shardedFile.readFilesWithRetries(), empty()); }
@Test public void testReadWithRetriesFailsWhenOutputDirEmpty() throws Exception { FilePatternMatchingShardedFile shardedFile = new FilePatternMatchingShardedFile(filePattern); thrown.expect(IOException.class); thrown.expectMessage(containsString("Unable to read file(s) after retrying")); shardedFile.readFilesWithRetries(fastClock, backOff); } }
@Test public void testReadWithRetriesFailsSinceFilesystemError() throws Exception { File tmpFile = tmpFolder.newFile(); Files.write("Test for file checksum verifier.", tmpFile, StandardCharsets.UTF_8); FilePatternMatchingShardedFile shardedFile = spy(new FilePatternMatchingShardedFile(filePattern)); doThrow(IOException.class).when(shardedFile).readLines(anyCollection()); thrown.expect(IOException.class); thrown.expectMessage(containsString("Unable to read file(s) after retrying")); shardedFile.readFilesWithRetries(fastClock, backOff); }
@Test public void testReadMultipleShardsWithoutShardNumber() throws Exception { String contents1 = "To be or not to be, "; String contents2 = "it is not a question."; File tmpFile1 = tmpFolder.newFile("result"); File tmpFile2 = tmpFolder.newFile("tmp"); Files.write(contents1, tmpFile1, StandardCharsets.UTF_8); Files.write(contents2, tmpFile2, StandardCharsets.UTF_8); FilePatternMatchingShardedFile shardedFile = new FilePatternMatchingShardedFile(filePattern); assertThat(shardedFile.readFilesWithRetries(), containsInAnyOrder(contents1, contents2)); }
@Test public void testReadMultipleShards() throws Exception { String contents1 = "To be or not to be, ", contents2 = "it is not a question.", contents3 = "should not be included"; File tmpFile1 = tmpFolder.newFile("result-000-of-002"); File tmpFile2 = tmpFolder.newFile("result-001-of-002"); File tmpFile3 = tmpFolder.newFile("tmp"); Files.write(contents1, tmpFile1, StandardCharsets.UTF_8); Files.write(contents2, tmpFile2, StandardCharsets.UTF_8); Files.write(contents3, tmpFile3, StandardCharsets.UTF_8); filePattern = LocalResources.fromFile(tmpFolder.getRoot(), true) .resolve("result-*", StandardResolveOptions.RESOLVE_FILE) .toString(); FilePatternMatchingShardedFile shardedFile = new FilePatternMatchingShardedFile(filePattern); assertThat(shardedFile.readFilesWithRetries(), containsInAnyOrder(contents1, contents2)); }