@Test public void testIgnoreCommitExceptionDuringRecovery() throws Exception { clock.setEpochMilli(0); harness.open(); harness.processElement("42", 0); final OperatorSubtaskState snapshot = harness.snapshot(0, 1); harness.notifyOfCompletedCheckpoint(1); throwException.set(true); closeTestHarness(); setUpTestHarness(); final long transactionTimeout = 1000; sinkFunction.setTransactionTimeout(transactionTimeout); sinkFunction.ignoreFailuresAfterTransactionTimeout(); try { harness.initializeState(snapshot); fail("Expected exception not thrown"); } catch (RuntimeException e) { assertEquals("Expected exception", e.getMessage()); } clock.setEpochMilli(transactionTimeout + 1); harness.initializeState(snapshot); assertExactlyOnce(Collections.singletonList("42")); }
@Test public void testLogTimeoutAlmostReachedWarningDuringRecovery() throws Exception { clock.setEpochMilli(0); final long transactionTimeout = 1000; final double warningRatio = 0.5; sinkFunction.setTransactionTimeout(transactionTimeout); sinkFunction.enableTransactionTimeoutWarnings(warningRatio); harness.open(); final OperatorSubtaskState snapshot = harness.snapshot(0, 1); final long elapsedTime = (long) ((double) transactionTimeout * warningRatio + 2); clock.setEpochMilli(elapsedTime); closeTestHarness(); setUpTestHarness(); sinkFunction.setTransactionTimeout(transactionTimeout); sinkFunction.enableTransactionTimeoutWarnings(warningRatio); harness.initializeState(snapshot); harness.open(); final List<String> logMessages = loggingEvents.stream().map(LoggingEvent::getRenderedMessage).collect(Collectors.toList()); closeTestHarness(); assertThat( logMessages, hasItem(containsString("has been open for 502 ms. " + "This is close to or even exceeding the transaction timeout of 1000 ms."))); }
@Test public void testFailBeforeNotify() throws Exception { harness.open(); harness.processElement("42", 0); harness.snapshot(0, 1); harness.processElement("43", 2); OperatorSubtaskState snapshot = harness.snapshot(1, 3); tmpDirectory.setWritable(false); try { harness.processElement("44", 4); harness.snapshot(2, 5); fail("something should fail"); } catch (Exception ex) { if (!(ex.getCause() instanceof ContentDump.NotWritableException)) { throw ex; } // ignore } closeTestHarness(); tmpDirectory.setWritable(true); setUpTestHarness(); harness.initializeState(snapshot); assertExactlyOnce(Arrays.asList("42", "43")); closeTestHarness(); assertEquals(0, tmpDirectory.listFiles().size()); }
harness.setup(); harness.initializeState( OperatorSnapshotUtil.getResourceFilename( "cep-migration-conditions-flink" + migrateVersion + "-snapshot"));
harness.setup(); harness.initializeState( OperatorSnapshotUtil.getResourceFilename( "cep-migration-single-pattern-afterwards-flink" + migrateVersion + "-snapshot"));
@Test public void testSnapshotAndRestore() throws Exception { LegacyKeyedProcessOperator<Integer, Integer, String> operator = new LegacyKeyedProcessOperator<>(new BothTriggeringFlatMapFunction()); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(5, 12L)); // snapshot and restore from scratch OperatorSubtaskState snapshot = testHarness.snapshot(0, 0); testHarness.close(); operator = new LegacyKeyedProcessOperator<>(new BothTriggeringFlatMapFunction()); testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); testHarness.setProcessingTime(5); testHarness.processWatermark(new Watermark(6)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("PROC:1777")); expectedOutput.add(new StreamRecord<>("EVENT:1777", 6L)); expectedOutput.add(new Watermark(6)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
@Test public void testSnapshotAndRestore() throws Exception { final int expectedKey = 5; KeyedProcessOperator<Integer, Integer, String> operator = new KeyedProcessOperator<>(new BothTriggeringFlatMapFunction(expectedKey)); OneInputStreamOperatorTestHarness<Integer, String> testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(expectedKey, 12L)); // snapshot and restore from scratch OperatorSubtaskState snapshot = testHarness.snapshot(0, 0); testHarness.close(); operator = new KeyedProcessOperator<>(new BothTriggeringFlatMapFunction(expectedKey)); testHarness = new KeyedOneInputStreamOperatorTestHarness<>(operator, new IdentityKeySelector<Integer>(), BasicTypeInfo.INT_TYPE_INFO); testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); testHarness.setProcessingTime(5); testHarness.processWatermark(new Watermark(6)); ConcurrentLinkedQueue<Object> expectedOutput = new ConcurrentLinkedQueue<>(); expectedOutput.add(new StreamRecord<>("PROC:1777")); expectedOutput.add(new StreamRecord<>("EVENT:1777", 6L)); expectedOutput.add(new Watermark(6)); TestHarnessUtil.assertOutputEquals("Output was not correct.", expectedOutput, testHarness.getOutput()); testHarness.close(); }
testHarness.initializeState( OperatorSnapshotUtil.getResourceFilename( "win-op-migration-test-apply-event-time-flink" + testMigrateVersion + "-snapshot"));
testHarness.initializeState( OperatorSnapshotUtil.getResourceFilename( "win-op-migration-test-reduce-event-time-flink" + testMigrateVersion + "-snapshot"));
harness.initializeState(snapshot); harness.open(); harness.initializeState(snapshot2); harness.open();
harness.initializeState(snapshot); harness.open(); harness.initializeState(snapshot2); harness.open();
testHarness.initializeState( OperatorSnapshotUtil.getResourceFilename( "win-op-migration-test-kryo-serialized-key-flink" + testMigrateVersion + "-snapshot"));
testHarness.initializeState( OperatorSnapshotUtil.getResourceFilename( "win-op-migration-test-apply-processing-time-flink" + testMigrateVersion + "-snapshot"));
testHarness.initializeState( OperatorSnapshotUtil.getResourceFilename( "win-op-migration-test-reduce-processing-time-flink" + testMigrateVersion + "-snapshot"));
harness = CepOperatorTestUtilities.getCepTestHarness(operator2); harness.setup(); harness.initializeState(snapshot); harness.open();
harness = CepOperatorTestUtilities.getCepTestHarness(operator2); harness.setup(); harness.initializeState(snapshot); harness.open();
testHarness.initializeState(latestSnapshot); testHarness.open();
@Test public void testSnapshotAndRestoreWrappedCheckpointedFunction() throws Exception { StreamMap<Integer, Integer> operator = new StreamMap<>( new WrappingTestFun(new WrappingTestFun(new InnerTestFun()))); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(5, 12L)); // snapshot and restore from scratch OperatorSubtaskState snapshot = testHarness.snapshot(0, 0); testHarness.close(); InnerTestFun innerTestFun = new InnerTestFun(); operator = new StreamMap<>(new WrappingTestFun(new WrappingTestFun(innerTestFun))); testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); Assert.assertTrue(innerTestFun.wasRestored); testHarness.close(); }
@Test public void testSnapshotAndRestoreWrappedListCheckpointed() throws Exception { StreamMap<Integer, Integer> operator = new StreamMap<>( new WrappingTestFun(new WrappingTestFun(new InnerTestFunList()))); OneInputStreamOperatorTestHarness<Integer, Integer> testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.open(); testHarness.processElement(new StreamRecord<>(5, 12L)); // snapshot and restore from scratch OperatorSubtaskState snapshot = testHarness.snapshot(0, 0); testHarness.close(); InnerTestFunList innerTestFun = new InnerTestFunList(); operator = new StreamMap<>(new WrappingTestFun(new WrappingTestFun(innerTestFun))); testHarness = new OneInputStreamOperatorTestHarness<>(operator); testHarness.setup(); testHarness.initializeState(snapshot); testHarness.open(); Assert.assertTrue(innerTestFun.wasRestored); testHarness.close(); }
testHarness.initializeState( OperatorSnapshotUtil.getResourceFilename( "win-op-migration-test-session-with-stateful-trigger-flink" + testMigrateVersion + "-snapshot"));