@Test public void testLogTimeoutAlmostReachedWarningDuringCommit() throws Exception { clock.setEpochMilli(0); final long transactionTimeout = 1000; final double warningRatio = 0.5; sinkFunction.setTransactionTimeout(transactionTimeout); sinkFunction.enableTransactionTimeoutWarnings(warningRatio); harness.open(); harness.snapshot(0, 1); final long elapsedTime = (long) ((double) transactionTimeout * warningRatio + 2); clock.setEpochMilli(elapsedTime); harness.notifyOfCompletedCheckpoint(1); final List<String> logMessages = loggingEvents.stream().map(LoggingEvent::getRenderedMessage).collect(Collectors.toList()); assertThat( logMessages, hasItem(containsString("has been open for 502 ms. " + "This is close to or even exceeding the transaction timeout of 1000 ms."))); }
testHarness.notifyOfCompletedCheckpoint(0L); // put the pending for 0 to the "committed" state TestUtils.checkLocalFs(outDir, 2, 2); testHarness.notifyOfCompletedCheckpoint(1L); // put the pending for 1 to the "committed" state TestUtils.checkLocalFs(outDir, 0, 4);
@Test public void testNotifyOfCompletedCheckpoint() throws Exception { harness.open(); harness.processElement("42", 0); harness.snapshot(0, 1); harness.processElement("43", 2); harness.snapshot(1, 3); harness.processElement("44", 4); harness.snapshot(2, 5); harness.notifyOfCompletedCheckpoint(1); assertExactlyOnce(Arrays.asList("42", "43")); assertEquals(2, tmpDirectory.listFiles().size()); // one for checkpointId 2 and second for the currentTransaction }
testHarness.notifyOfCompletedCheckpoint(2L); TestUtils.checkLocalFs(outDir, 2, 1);
testHarness.notifyOfCompletedCheckpoint(2L);
testHarness.notifyOfCompletedCheckpoint(1L); // the pending for checkpoint 1 are committed TestUtils.checkLocalFs(outDir, 3, 1); testHarness.notifyOfCompletedCheckpoint(3L); // all the pending for checkpoint 2 and 3 are committed TestUtils.checkLocalFs(outDir, 0, 4);
testHarness.notifyOfCompletedCheckpoint(0); testHarness.notifyOfCompletedCheckpoint(1); testHarness.notifyOfCompletedCheckpoint(2);
@Test public void testIdealCircumstances() throws Exception { S sink = createSink(); OneInputStreamOperatorTestHarness<IN, IN> testHarness = new OneInputStreamOperatorTestHarness<>(sink); testHarness.open(); int elementCounter = 1; int snapshotCount = 0; for (int x = 0; x < 20; x++) { testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0))); elementCounter++; } testHarness.snapshot(snapshotCount++, 0); testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1); for (int x = 0; x < 20; x++) { testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 1))); elementCounter++; } testHarness.snapshot(snapshotCount++, 0); testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1); for (int x = 0; x < 20; x++) { testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 2))); elementCounter++; } testHarness.snapshot(snapshotCount++, 0); testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1); verifyResultsIdealCircumstances(sink); }
@Test public void testIgnoreCommitExceptionDuringRecovery() throws Exception { clock.setEpochMilli(0); harness.open(); harness.processElement("42", 0); final OperatorSubtaskState snapshot = harness.snapshot(0, 1); harness.notifyOfCompletedCheckpoint(1); throwException.set(true); closeTestHarness(); setUpTestHarness(); final long transactionTimeout = 1000; sinkFunction.setTransactionTimeout(transactionTimeout); sinkFunction.ignoreFailuresAfterTransactionTimeout(); try { harness.initializeState(snapshot); fail("Expected exception not thrown"); } catch (RuntimeException e) { assertEquals("Expected exception", e.getMessage()); } clock.setEpochMilli(transactionTimeout + 1); harness.initializeState(snapshot); assertExactlyOnce(Collections.singletonList("42")); }
testHarness.notifyOfCompletedCheckpoint(2L); TestUtils.checkLocalFs(outDir, 0, 2);
@Test public void testDataPersistenceUponMissedNotify() throws Exception { S sink = createSink(); OneInputStreamOperatorTestHarness<IN, IN> testHarness = new OneInputStreamOperatorTestHarness<>(sink); testHarness.open(); int elementCounter = 1; int snapshotCount = 0; for (int x = 0; x < 20; x++) { testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 0))); elementCounter++; } testHarness.snapshot(snapshotCount++, 0); testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1); for (int x = 0; x < 20; x++) { testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 1))); elementCounter++; } testHarness.snapshot(snapshotCount++, 0); for (int x = 0; x < 20; x++) { testHarness.processElement(new StreamRecord<>(generateValue(elementCounter, 2))); elementCounter++; } testHarness.snapshot(snapshotCount++, 0); testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1); verifyResultsDataPersistenceUponMissedNotify(sink); }
testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1); testHarness.notifyOfCompletedCheckpoint(snapshotCount - 1);
testHarness2.open(); testHarness2.notifyOfCompletedCheckpoint(snapshotCount); testHarness3.notifyOfCompletedCheckpoint(snapshotCount);
@Test public void testClosingOnSnapshot() throws Exception { final File outDir = TEMP_FOLDER.newFolder(); try ( OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> testHarness = TestUtils.createRescalingTestSink(outDir, 1, 0, 100L, 2L) ) { testHarness.setup(); testHarness.open(); testHarness.setProcessingTime(0L); testHarness.processElement(new StreamRecord<>(Tuple2.of("test1", 1), 1L)); testHarness.processElement(new StreamRecord<>(Tuple2.of("test2", 1), 1L)); TestUtils.checkLocalFs(outDir, 2, 0); // this is to check the inactivity threshold testHarness.setProcessingTime(101L); TestUtils.checkLocalFs(outDir, 2, 0); testHarness.processElement(new StreamRecord<>(Tuple2.of("test3", 1), 1L)); TestUtils.checkLocalFs(outDir, 3, 0); testHarness.snapshot(0L, 1L); TestUtils.checkLocalFs(outDir, 3, 0); testHarness.notifyOfCompletedCheckpoint(0L); TestUtils.checkLocalFs(outDir, 0, 3); testHarness.snapshot(1L, 0L); testHarness.processElement(new StreamRecord<>(Tuple2.of("test4", 10), 10L)); TestUtils.checkLocalFs(outDir, 1, 3); } // at close it is not moved to final. TestUtils.checkLocalFs(outDir, 1, 3); }
mergedTestHarness.notifyOfCompletedCheckpoint(snapshotCount);