private void setUpTestHarness() throws Exception { sinkFunction = new ContentDumpSinkFunction(); harness = new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sinkFunction), StringSerializer.INSTANCE); harness.setup(); }
@Override public final void setChainingStrategy(ChainingStrategy strategy) { operator.setChainingStrategy(strategy); } }
@Override public void open() throws Exception { super.open(); this.sinkContext = new SimpleContext<>(getProcessingTimeService()); }
inputStream.addSink(kafkaSink.getUserFunction());
inputStream.addSink(kafkaSink.getUserFunction()); } else { kafkaServer.produceIntoKafka(inputStream, topic + i, keyedSerializationSchema, properties, partitioner);
@Override public <T> StreamSink<T> getProducerSink(String topic, KeyedSerializationSchema<T> serSchema, Properties props, FlinkKafkaPartitioner<T> partitioner) { return new StreamSink<>(new FlinkKafkaProducer011<>( topic, serSchema, props, Optional.ofNullable(partitioner), producerSemantic, FlinkKafkaProducer011.DEFAULT_KAFKA_PRODUCERS_POOL_SIZE)); }
@Override public final void setChainingStrategy(ChainingStrategy strategy) { operator.setChainingStrategy(strategy); } }
@Override public void open() throws Exception { super.open(); this.sinkContext = new SimpleContext<>(getProcessingTimeService()); }
@Override public <T> StreamSink<T> getProducerSink(String topic, KeyedSerializationSchema<T> serSchema, Properties props, FlinkKafkaPartitioner<T> partitioner) { return new StreamSink<>(new FlinkKafkaProducer<T>( topic, serSchema, props, Optional.ofNullable(partitioner), producerSemantic, FlinkKafkaProducer.DEFAULT_KAFKA_PRODUCERS_POOL_SIZE)); }
@Override public final void setChainingStrategy(ChainingStrategy strategy) { operator.setChainingStrategy(strategy); } }
@Override public void open() throws Exception { super.open(); this.sinkContext = new SimpleContext<>(getProcessingTimeService()); }
@Override public <T> StreamSink<T> getProducerSink( String topic, KeyedSerializationSchema<T> serSchema, Properties props, FlinkKafkaPartitioner<T> partitioner) { FlinkKafkaProducer09<T> prod = new FlinkKafkaProducer09<>(topic, serSchema, props, partitioner); prod.setFlushOnCheckpoint(true); return new StreamSink<>(prod); }
@Override public final void setChainingStrategy(ChainingStrategy strategy) { operator.setChainingStrategy(strategy); } }
@Override public <T> StreamSink<T> getProducerSink(String topic, KeyedSerializationSchema<T> serSchema, Properties props, FlinkKafkaPartitioner<T> partitioner) { FlinkKafkaProducer010<T> prod = new FlinkKafkaProducer010<>(topic, serSchema, props, partitioner); prod.setFlushOnCheckpoint(true); return new StreamSink<>(prod); }
static OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> createTestSinkWithBulkEncoder( final File outDir, final int totalParallelism, final int taskIdx, final long bucketCheckInterval, final BucketAssigner<Tuple2<String, Integer>, String> bucketer, final BulkWriter.Factory<Tuple2<String, Integer>> writer, final BucketFactory<Tuple2<String, Integer>, String> bucketFactory) throws Exception { StreamingFileSink<Tuple2<String, Integer>> sink = StreamingFileSink .forBulkFormat(new Path(outDir.toURI()), writer) .withBucketAssigner(bucketer) .withBucketCheckInterval(bucketCheckInterval) .withBucketFactory(bucketFactory) .build(); return new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink), 10, totalParallelism, taskIdx); }
static OneInputStreamOperatorTestHarness<Tuple2<String, Integer>, Object> createCustomRescalingTestSink( final File outDir, final int totalParallelism, final int taskIdx, final long bucketCheckInterval, final BucketAssigner<Tuple2<String, Integer>, String> bucketer, final Encoder<Tuple2<String, Integer>> writer, final RollingPolicy<Tuple2<String, Integer>, String> rollingPolicy, final BucketFactory<Tuple2<String, Integer>, String> bucketFactory) throws Exception { StreamingFileSink<Tuple2<String, Integer>> sink = StreamingFileSink .forRowFormat(new Path(outDir.toURI()), writer) .withBucketAssigner(bucketer) .withRollingPolicy(rollingPolicy) .withBucketCheckInterval(bucketCheckInterval) .withBucketFactory(bucketFactory) .build(); return new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink), 10, totalParallelism, taskIdx); }
/** Tests that any item failure in the listener callbacks is rethrown on an immediately following checkpoint. */ @Test public void testItemFailureRethrownOnCheckpoint() throws Throwable { final DummyElasticsearchSink<String> sink = new DummyElasticsearchSink<>( new HashMap<String, String>(), new SimpleSinkFunction<String>(), new NoOpFailureHandler()); final OneInputStreamOperatorTestHarness<String, Object> testHarness = new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink)); testHarness.open(); // setup the next bulk request, and its mock item failures sink.setMockItemFailuresListForNextBulkItemResponses(Collections.singletonList(new Exception("artificial failure for record"))); testHarness.processElement(new StreamRecord<>("msg")); verify(sink.getMockBulkProcessor(), times(1)).add(any(IndexRequest.class)); // manually execute the next bulk request sink.manualBulkRequestWithAllPendingRequests(); try { testHarness.snapshot(1L, 1000L); } catch (Exception e) { // the snapshot should have failed with the failure Assert.assertTrue(e.getCause().getCause().getMessage().contains("artificial failure for record")); // test succeeded return; } Assert.fail(); }
/** Tests that any bulk failure in the listener callbacks is rethrown on an immediately following checkpoint. */ @Test public void testBulkFailureRethrownOnCheckpoint() throws Throwable { final DummyElasticsearchSink<String> sink = new DummyElasticsearchSink<>( new HashMap<String, String>(), new SimpleSinkFunction<String>(), new NoOpFailureHandler()); final OneInputStreamOperatorTestHarness<String, Object> testHarness = new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink)); testHarness.open(); // setup the next bulk request, and let the whole bulk request fail sink.setFailNextBulkRequestCompletely(new Exception("artificial failure for bulk request")); testHarness.processElement(new StreamRecord<>("msg")); verify(sink.getMockBulkProcessor(), times(1)).add(any(IndexRequest.class)); // manually execute the next bulk request sink.manualBulkRequestWithAllPendingRequests(); try { testHarness.snapshot(1L, 1000L); } catch (Exception e) { // the snapshot should have failed with the bulk request failure Assert.assertTrue(e.getCause().getCause().getMessage().contains("artificial failure for bulk request")); // test succeeded return; } Assert.fail(); }
/** Tests that any item failure in the listener callbacks is rethrown on an immediately following invoke call. */ @Test public void testItemFailureRethrownOnInvoke() throws Throwable { final DummyElasticsearchSink<String> sink = new DummyElasticsearchSink<>( new HashMap<String, String>(), new SimpleSinkFunction<String>(), new NoOpFailureHandler()); final OneInputStreamOperatorTestHarness<String, Object> testHarness = new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink)); testHarness.open(); // setup the next bulk request, and its mock item failures sink.setMockItemFailuresListForNextBulkItemResponses(Collections.singletonList(new Exception("artificial failure for record"))); testHarness.processElement(new StreamRecord<>("msg")); verify(sink.getMockBulkProcessor(), times(1)).add(any(IndexRequest.class)); // manually execute the next bulk request sink.manualBulkRequestWithAllPendingRequests(); try { testHarness.processElement(new StreamRecord<>("next msg")); } catch (Exception e) { // the invoke should have failed with the failure Assert.assertTrue(e.getCause().getMessage().contains("artificial failure for record")); // test succeeded return; } Assert.fail(); }
/** * This test is meant to assure that testAtLeastOnceSink is valid by testing that if flushing is disabled, * the snapshot method does indeed finishes without waiting for pending requests; * we set a timeout because the test will not finish if the logic is broken. */ @Test(timeout = 5000) public void testDoesNotWaitForPendingRequestsIfFlushingDisabled() throws Exception { final DummyElasticsearchSink<String> sink = new DummyElasticsearchSink<>( new HashMap<String, String>(), new SimpleSinkFunction<String>(), new DummyRetryFailureHandler()); sink.disableFlushOnCheckpoint(); // disable flushing final OneInputStreamOperatorTestHarness<String, Object> testHarness = new OneInputStreamOperatorTestHarness<>(new StreamSink<>(sink)); testHarness.open(); // setup the next bulk request, and let bulk request succeed sink.setMockItemFailuresListForNextBulkItemResponses(Collections.singletonList(new Exception("artificial failure for record"))); testHarness.processElement(new StreamRecord<>("msg-1")); verify(sink.getMockBulkProcessor(), times(1)).add(any(IndexRequest.class)); // the snapshot should not block even though we haven't flushed the bulk request testHarness.snapshot(1L, 1000L); testHarness.close(); }