@Test public void testGenerateRestart() throws Exception { Topology t = newTopology("testGenerateRestart"); Path tempFile1 = FileUtil.createTempFile("test1", ".txt", getLines()); System.out.println("Test: "+t.getName()+" "+tempFile1); ProcessBuilder cmd = new ProcessBuilder(mkCatFileCmd(tempFile1.toString())); int NUM_RUNS = 3; List<String> expLines = new ArrayList<>(); for (int i = 0; i < NUM_RUNS; i++) { expLines.addAll(Arrays.asList(getLines())); } // N.B. if looking at trace: EDGENT-224 generate() continues running after job is closed TStream<String> s = CommandStreams.generate(t, cmd); completeAndValidate("", t, s, 10 + ((NUM_RUNS-1) * 1/*restart delay time*/), expLines.toArray(new String[0])); }
@Test public void testTextFileReader() throws Exception { Topology t = newTopology("testTextFileReader"); String[] lines = getLines(); String[] ucLines = toUpperCase(lines); String[] allLines = concat(lines, ucLines); Path tempFile1 = FileUtil.createTempFile("test1", "txt", lines); Path tempFile2 = FileUtil.createTempFile("test2", "txt", ucLines); TStream<String> contents = FileStreams.textFileReader( t.strings(tempFile1.toAbsolutePath().toString(), tempFile2.toAbsolutePath().toString())); try { completeAndValidate("", t, contents, 10, allLines); } finally { tempFile1.toFile().delete(); tempFile2.toFile().delete(); } }
@Test public void testTextFileReaderProblemPaths() throws Exception { Topology t = newTopology("testTextFileReaderProblemPaths"); String[] lines = getLines(); String[] ucLines = toUpperCase(lines); String[] allLines = concat(lines, ucLines); Path tempFile1 = FileUtil.createTempFile("test1", "txt", lines); Path tempFile2 = FileUtil.createTempFile("test2", "txt", ucLines); // ensure a problem in one file (tuple) doesn't affect others. // The problem files should result in a log entry but otherwise be ignored. TStream<String> contents = FileStreams.textFileReader( t.strings(tempFile1.toAbsolutePath().toString(), "/no-such-file", "/tmp", tempFile2.toAbsolutePath().toString())); try { completeAndValidate("", t, contents, 10, allLines); } finally { tempFile1.toFile().delete(); tempFile2.toFile().delete(); } }
@Test public void testPeriodicSource() throws Exception { Topology t = newTopology("testPeriodicSource"); Path tempFile1 = FileUtil.createTempFile("test1", ".txt", getLines()); System.out.println("Test: "+t.getName()+" "+tempFile1); ProcessBuilder cmd = new ProcessBuilder(mkCatFileCmd(tempFile1.toString())); int NUM_POLLS = 3; List<String> expLines = new ArrayList<>(); for (int i = 0; i < NUM_POLLS; i++) { expLines.addAll(Arrays.asList(getLines())); } TStream<List<String>> ls = CommandStreams.periodicSource(t, cmd, 1, TimeUnit.SECONDS); TStream<String> s = ls.flatMap(list -> list); try { completeAndValidate("", t, s, 10, expLines.toArray(new String[0])); } finally { tempFile1.toFile().delete(); } }
@Test public void testNoFilesCreated() throws Exception { // complete before any files are generated Topology t = newTopology("testNoFilesCreated"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); // build expected results List<List<String>> expResults = Collections.emptyList(); TStream<String> s = t.events(eventSetup -> { /* no tuples generated */ }); FileStreams.textFileWriter(s, () -> basePath.toString()); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testGenerate() throws Exception { Topology t = newTopology("testGenerate"); Path tempFile1 = FileUtil.createTempFile("test1", ".txt", getLines()); System.out.println("Test: "+t.getName()+" "+tempFile1); ProcessBuilder cmd = new ProcessBuilder(mkCatFileCmd(tempFile1.toString())); // N.B. if looking at trace: EDGENT-224 generate() continues running after job is closed TStream<String> s = CommandStreams.generate(t, cmd); try { completeAndValidate("", t, s, 10, getLines()); } finally { tempFile1.toFile().delete(); } }
@Test public void testFlushTupleBased() throws Exception { Topology t = newTopology("testFlushTupleBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newPredicateBasedConfig( tuple -> tuple.startsWith("1-") || tuple.startsWith("3-")), FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), // all in 1 file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testCycleTupleBased() throws Exception { Topology t = newTopology("testCycleTupleBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // a tuple based config / predicate. in this case should end up with 3 files. // flush on the last tuple too to ensure the test completes before TMO. Predicate<String> cycleIt = tuple -> tuple.startsWith("1-") || tuple.startsWith("3-") || tuple.equals(lines[lines.length-1]); List<List<String>> expResults = buildExpResults(lines, cycleIt); assertEquals(3, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newPredicateBasedConfig(cycleIt), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testFlushImplicit() throws Exception { Topology t = newTopology("testFlushImplicit"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testFlushCntBased() throws Exception { Topology t = newTopology("testFlushCntBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newCountBasedConfig(1), // every tuple FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), // all in 1 file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testRetainAggSizeBased() throws Exception { // more aggsize than configured; only keep aggsize worth Topology t = newTopology("testRetainAggSizeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuple per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); // agg size only enough for last two lines long aggregateFileSize = 2 * (("1-"+getStr()).getBytes(StandardCharsets.UTF_8).length + 1/*eol*/); expResults.remove(0); expResults.remove(0); assertEquals(2, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(1), FileWriterRetentionConfig.newAggregateFileSizeBasedConfig(aggregateFileSize) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testOneFileCreated() throws Exception { // all lines into a single (the first) file Topology t = newTopology("testOneFileCreated"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); assertEquals(1, expResults.size()); TStream<String> s = t.strings(lines); // default writer policy TSink<String> sink = FileStreams.textFileWriter(s, () -> basePath.toString()); // note, with only 4 tuples, default policy won't cycle (finalize the cur file) // to make the expResults present until job stops (TMO) completeAndValidateWriter(t, TMO_SEC, basePath, expResults); assertNotNull(sink); }
@Test public void testCompressedFileWriterPolicy() throws Exception { Topology t = newTopology("testCompressedFileWriterPolicy"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net 2 tuples per file int cntTuples = 2; AtomicInteger cnt = new AtomicInteger(); Predicate<String> cycleIt = tuple -> cnt.incrementAndGet() % cntTuples == 0; List<List<String>> expResults = buildExpResults(lines, cycleIt); assertEquals(lines.length / cntTuples, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new CompressedFileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(cntTuples), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testAllTimeBased() throws Exception { // exercise case with multiple timer based policies Topology t = newTopology("testAllTimeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // keep all given age and TMO_SEC int ageSec = 10; long periodMsec = TimeUnit.SECONDS.toMillis(1); // net one tuple per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newTimeBasedConfig(TimeUnit.MILLISECONDS.toMillis(250)), FileWriterCycleConfig.newConfig(1, 2000, TimeUnit.SECONDS.toMillis(10), null), FileWriterRetentionConfig.newAgeBasedConfig(ageSec, periodMsec) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testFlushTimeBased() throws Exception { Topology t = newTopology("testFlushTimeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); // add delay so time flush happens int throttleSec = 1; TStream<String> s = PlumbingStreams.blockingThrottle( t.strings(lines), throttleSec, TimeUnit.SECONDS); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newTimeBasedConfig(TimeUnit.MILLISECONDS.toMillis(250)), FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), // all in 1 file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, (lines.length*throttleSec)+TMO_SEC, basePath, expResults); }
@Test public void testCycleCntBased() throws Exception { Topology t = newTopology("testCycleCntBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net two tuples per file int cntTuples = 2; AtomicInteger cnt = new AtomicInteger(); Predicate<String> cycleIt = tuple -> cnt.incrementAndGet() % cntTuples == 0; List<List<String>> expResults = buildExpResults(lines, cycleIt); assertEquals(lines.length / cntTuples, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(cntTuples), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testCycleTimeBased() throws Exception { Topology t = newTopology("testCycleTimeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuple per file with 250msec cycle config and 1 throttle List<List<String>> expResults = buildExpResults(lines, tuple -> true); // add delay so time cycle happens // also verifies only cycle if there's something to cycle // (i.e., these cycles happen faster than tuples are written) int throttleSec = 1; TStream<String> s = PlumbingStreams.blockingThrottle( t.strings(lines), throttleSec, TimeUnit.SECONDS); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newTimeBasedConfig(TimeUnit.MILLISECONDS.toMillis(250)), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, (lines.length*throttleSec)+TMO_SEC, basePath, expResults); }
@Test public void testManyFiles() throws Exception { Topology t = newTopology("testManyFiles"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuples per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); // in this config files are create very fast hence they end // up exercising the _<n> suffix to basePath_YYYYMMDD_HHMMSS TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), // no extra flush FileWriterCycleConfig.newCountBasedConfig(1), // yield one line per file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testCycleSizeBased() throws Exception { Topology t = newTopology("testCycleSizeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuple per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); int fileSize = 2; TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newFileSizeBasedConfig(fileSize), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testManyFilesSlow() throws Exception { Topology t = newTopology("testManyFilesSlow"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuples per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); // add delay so we get different files w/o a _<n> suffix int throttleSec = 2; TStream<String> s = PlumbingStreams.blockingThrottle( t.strings(lines), throttleSec, TimeUnit.SECONDS); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), // no extra flush FileWriterCycleConfig.newCountBasedConfig(1), // yield one line per file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, (lines.length*throttleSec)+TMO_SEC, basePath, expResults); }