@Test public void testGenerateRestart() throws Exception { Topology t = newTopology("testGenerateRestart"); Path tempFile1 = FileUtil.createTempFile("test1", ".txt", getLines()); System.out.println("Test: "+t.getName()+" "+tempFile1); ProcessBuilder cmd = new ProcessBuilder(mkCatFileCmd(tempFile1.toString())); int NUM_RUNS = 3; List<String> expLines = new ArrayList<>(); for (int i = 0; i < NUM_RUNS; i++) { expLines.addAll(Arrays.asList(getLines())); } // N.B. if looking at trace: EDGENT-224 generate() continues running after job is closed TStream<String> s = CommandStreams.generate(t, cmd); completeAndValidate("", t, s, 10 + ((NUM_RUNS-1) * 1/*restart delay time*/), expLines.toArray(new String[0])); }
/** * Validate that the file contains the specified content. * @param path file to validate * @param lines the expected content * @throws Exception on failure */ public static void validateFile(Path path, String[] lines) throws Exception { validateFile(path, lines, false); }
Topology t = newTopology("testSink"); Path tempFile1 = FileUtil.createTempFile("test1", ".txt", new String[0]); System.out.println("Test: "+t.getName()+" "+tempFile1); FileUtil.validateFile(tempFile1, getLines());
@Test public void testTextFileReader() throws Exception { Topology t = newTopology("testTextFileReader"); String[] lines = getLines(); String[] ucLines = toUpperCase(lines); String[] allLines = concat(lines, ucLines); Path tempFile1 = FileUtil.createTempFile("test1", "txt", lines); Path tempFile2 = FileUtil.createTempFile("test2", "txt", ucLines); TStream<String> contents = FileStreams.textFileReader( t.strings(tempFile1.toAbsolutePath().toString(), tempFile2.toAbsolutePath().toString())); try { completeAndValidate("", t, contents, 10, allLines); } finally { tempFile1.toFile().delete(); tempFile2.toFile().delete(); } }
public boolean valid() { try { return FileUtil.validateFile(tempFile1, getLines(), true); } catch (Exception e) { return false; } } public Object getResult() { return "todo-files-lines"; }
@Test public void testTextFileReaderProblemPaths() throws Exception { Topology t = newTopology("testTextFileReaderProblemPaths"); String[] lines = getLines(); String[] ucLines = toUpperCase(lines); String[] allLines = concat(lines, ucLines); Path tempFile1 = FileUtil.createTempFile("test1", "txt", lines); Path tempFile2 = FileUtil.createTempFile("test2", "txt", ucLines); // ensure a problem in one file (tuple) doesn't affect others. // The problem files should result in a log entry but otherwise be ignored. TStream<String> contents = FileStreams.textFileReader( t.strings(tempFile1.toAbsolutePath().toString(), "/no-such-file", "/tmp", tempFile2.toAbsolutePath().toString())); try { completeAndValidate("", t, contents, 10, allLines); } finally { tempFile1.toFile().delete(); tempFile2.toFile().delete(); } }
@Test public void testPeriodicSource() throws Exception { Topology t = newTopology("testPeriodicSource"); Path tempFile1 = FileUtil.createTempFile("test1", ".txt", getLines()); System.out.println("Test: "+t.getName()+" "+tempFile1); ProcessBuilder cmd = new ProcessBuilder(mkCatFileCmd(tempFile1.toString())); int NUM_POLLS = 3; List<String> expLines = new ArrayList<>(); for (int i = 0; i < NUM_POLLS; i++) { expLines.addAll(Arrays.asList(getLines())); } TStream<List<String>> ls = CommandStreams.periodicSource(t, cmd, 1, TimeUnit.SECONDS); TStream<String> s = ls.flatMap(list -> list); try { completeAndValidate("", t, s, 10, expLines.toArray(new String[0])); } finally { tempFile1.toFile().delete(); } }
@Test public void testNoFilesCreated() throws Exception { // complete before any files are generated Topology t = newTopology("testNoFilesCreated"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); // build expected results List<List<String>> expResults = Collections.emptyList(); TStream<String> s = t.events(eventSetup -> { /* no tuples generated */ }); FileStreams.textFileWriter(s, () -> basePath.toString()); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testGenerate() throws Exception { Topology t = newTopology("testGenerate"); Path tempFile1 = FileUtil.createTempFile("test1", ".txt", getLines()); System.out.println("Test: "+t.getName()+" "+tempFile1); ProcessBuilder cmd = new ProcessBuilder(mkCatFileCmd(tempFile1.toString())); // N.B. if looking at trace: EDGENT-224 generate() continues running after job is closed TStream<String> s = CommandStreams.generate(t, cmd); try { completeAndValidate("", t, s, 10, getLines()); } finally { tempFile1.toFile().delete(); } }
String[] ucLines = toUpperCase(lines); Path tempFile1 = FileUtil.createTempFile("test1", "txt", lines); Path tempFile2 = FileUtil.createTempFile("test2", "txt", ucLines);
@Test public void testFlushTupleBased() throws Exception { Topology t = newTopology("testFlushTupleBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newPredicateBasedConfig( tuple -> tuple.startsWith("1-") || tuple.startsWith("3-")), FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), // all in 1 file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testCycleTupleBased() throws Exception { Topology t = newTopology("testCycleTupleBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // a tuple based config / predicate. in this case should end up with 3 files. // flush on the last tuple too to ensure the test completes before TMO. Predicate<String> cycleIt = tuple -> tuple.startsWith("1-") || tuple.startsWith("3-") || tuple.equals(lines[lines.length-1]); List<List<String>> expResults = buildExpResults(lines, cycleIt); assertEquals(3, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newPredicateBasedConfig(cycleIt), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testFlushImplicit() throws Exception { Topology t = newTopology("testFlushImplicit"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testFlushCntBased() throws Exception { Topology t = newTopology("testFlushCntBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newCountBasedConfig(1), // every tuple FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), // all in 1 file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testRetainAggSizeBased() throws Exception { // more aggsize than configured; only keep aggsize worth Topology t = newTopology("testRetainAggSizeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net one tuple per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); // agg size only enough for last two lines long aggregateFileSize = 2 * (("1-"+getStr()).getBytes(StandardCharsets.UTF_8).length + 1/*eol*/); expResults.remove(0); expResults.remove(0); assertEquals(2, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(1), FileWriterRetentionConfig.newAggregateFileSizeBasedConfig(aggregateFileSize) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testOneFileCreated() throws Exception { // all lines into a single (the first) file Topology t = newTopology("testOneFileCreated"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); assertEquals(1, expResults.size()); TStream<String> s = t.strings(lines); // default writer policy TSink<String> sink = FileStreams.textFileWriter(s, () -> basePath.toString()); // note, with only 4 tuples, default policy won't cycle (finalize the cur file) // to make the expResults present until job stops (TMO) completeAndValidateWriter(t, TMO_SEC, basePath, expResults); assertNotNull(sink); }
@Test public void testCompressedFileWriterPolicy() throws Exception { Topology t = newTopology("testCompressedFileWriterPolicy"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net 2 tuples per file int cntTuples = 2; AtomicInteger cnt = new AtomicInteger(); Predicate<String> cycleIt = tuple -> cnt.incrementAndGet() % cntTuples == 0; List<List<String>> expResults = buildExpResults(lines, cycleIt); assertEquals(lines.length / cntTuples, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new CompressedFileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(cntTuples), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testAllTimeBased() throws Exception { // exercise case with multiple timer based policies Topology t = newTopology("testAllTimeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // keep all given age and TMO_SEC int ageSec = 10; long periodMsec = TimeUnit.SECONDS.toMillis(1); // net one tuple per file List<List<String>> expResults = buildExpResults(lines, tuple -> true); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newTimeBasedConfig(TimeUnit.MILLISECONDS.toMillis(250)), FileWriterCycleConfig.newConfig(1, 2000, TimeUnit.SECONDS.toMillis(10), null), FileWriterRetentionConfig.newAgeBasedConfig(ageSec, periodMsec) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }
@Test public void testFlushTimeBased() throws Exception { Topology t = newTopology("testFlushTimeBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net all in one, the first, file List<List<String>> expResults = buildExpResults(lines, tuple -> false); // add delay so time flush happens int throttleSec = 1; TStream<String> s = PlumbingStreams.blockingThrottle( t.strings(lines), throttleSec, TimeUnit.SECONDS); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newTimeBasedConfig(TimeUnit.MILLISECONDS.toMillis(250)), FileWriterCycleConfig.newCountBasedConfig(expResults.get(0).size()), // all in 1 file FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, (lines.length*throttleSec)+TMO_SEC, basePath, expResults); }
@Test public void testCycleCntBased() throws Exception { Topology t = newTopology("testCycleCntBased"); // establish a base path Path basePath = createTempFile("test1", "txt", new String[0]); String[] lines = getLines(); // build expected results // net two tuples per file int cntTuples = 2; AtomicInteger cnt = new AtomicInteger(); Predicate<String> cycleIt = tuple -> cnt.incrementAndGet() % cntTuples == 0; List<List<String>> expResults = buildExpResults(lines, cycleIt); assertEquals(lines.length / cntTuples, expResults.size()); TStream<String> s = t.strings(lines); IFileWriterPolicy<String> policy = new FileWriterPolicy<String>( FileWriterFlushConfig.newImplicitConfig(), FileWriterCycleConfig.newCountBasedConfig(cntTuples), FileWriterRetentionConfig.newFileCountBasedConfig(10) ); FileStreams.textFileWriter(s, () -> basePath.toString(), () -> policy); completeAndValidateWriter(t, TMO_SEC, basePath, expResults); }