@Override public PDone expand(PCollection<V> input) { return input .apply( "Kafka values with default key", MapElements.via( new SimpleFunction<V, KV<K, V>>() { @Override public KV<K, V> apply(V element) { return KV.of(null, element); } })) .setCoder(KvCoder.of(new NullOnlyCoder<>(), input.getCoder())) .apply(kvWriteTransform); }
@Test(expected = NullPointerException.class) public void testJoinNullValueIsNull() { p.enableAbandonedNodeEnforcement(false); Join.rightOuterJoin( p.apply("CreateLeft", Create.empty(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()))), p.apply( "CreateRight", Create.empty(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))), null); } }
@Test public void noInputUnkeyedOutput() { PCollection<KV<Integer, Iterable<Void>>> unkeyed = p.apply( Create.of(KV.<Integer, Iterable<Void>>of(-1, Collections.emptyList())) .withCoder(KvCoder.of(VarIntCoder.of(), IterableCoder.of(VoidCoder.of())))); p.traverseTopologically(visitor); assertThat(visitor.getKeyedPValues(), not(hasItem(unkeyed))); }
@Test(expected = NullPointerException.class) public void testJoinLeftNullValueIsNull() { p.enableAbandonedNodeEnforcement(false); Join.fullOuterJoin( p.apply("CreateLeft", Create.empty(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()))), p.apply( "CreateRight", Create.empty(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))), null, ""); }
@Test public void invalidSideInputThrowsException() { ExecutableStage stage = createExecutableStage(Collections.emptyList()); FlinkBatchSideInputHandlerFactory factory = FlinkBatchSideInputHandlerFactory.forStage(stage, context); thrown.expect(instanceOf(IllegalArgumentException.class)); factory.forSideInput( "transform-id", "side-input", MULTIMAP_ACCESS, KvCoder.of(VoidCoder.of(), VoidCoder.of()), GlobalWindow.Coder.INSTANCE); }
@Override public PCollection<KV<String, String>> expand(PCollection<String> input) { checkArgument(connectionConfiguration() != null, "withConnectionConfiguration() is required"); return input .apply(ParDo.of(new ReadFn(connectionConfiguration(), batchSize()))) .setCoder(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of())) .apply(new Reparallelize()); } }
@Test(expected = NullPointerException.class) public void testJoinNullValueIsNull() { p.enableAbandonedNodeEnforcement(false); Join.leftOuterJoin( p.apply("CreateLeft", Create.empty(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()))), p.apply( "CreateRight", Create.empty(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))), null); } }
@Test public void testTypeOverSpecifiedWithMultipleCoders() throws Exception { thrown.expect(CannotProvideCoderException.class); thrown.expectMessage("type is over specified"); CoderRegistry.createDefault() .getCoder( new TypeDescriptor<Integer>() {}, new TypeDescriptor<KV<Integer, Integer>>() {}, KvCoder.of(BigEndianIntegerCoder.of(), VarIntCoder.of())); }
@Override public PDone expand(PCollection<V> input) { return input .apply( "Kafka values with default key", MapElements.via( new SimpleFunction<V, KV<K, V>>() { @Override public KV<K, V> apply(V element) { return KV.of(null, element); } })) .setCoder(KvCoder.of(new NullOnlyCoder<>(), input.getCoder())) .apply(kvWriteTransform); }
public static final void main(String args[]) throws Exception { PipelineOptions options = PipelineOptionsFactory.fromArgs(args).create(); Pipeline pipeline = Pipeline.create(options); pipeline .apply(JdbcIO.<KV<Long, String>>read() .withDataSourceConfiguration(JdbcIO.DataSourceConfiguration.create("org.apache.derby.jdbc.ClientDriver", "jdbc:derby://localhost:1527/beam")) .withQuery("select * from artist") .withRowMapper(new JdbcIO.RowMapper<KV<Long, String>>() { @Override public KV<Long, String> mapRow(ResultSet resultSet) throws Exception { KV<Long, String> kv = KV.of(System.currentTimeMillis(), resultSet.getString("name") + "," + resultSet.getString("label")); return kv; } }) .withCoder(KvCoder.of(SerializableCoder.of(Long.class), StringUtf8Coder.of()))) .apply(KafkaIO.<Long, String>write().withBootstrapServers("localhost:9092").withTopic("BEAM").withKeySerializer(LongSerializer.class).withValueSerializer(StringSerializer.class)); pipeline.run(); }
@Test(expected = NullPointerException.class) public void testJoinRightNullValueIsNull() { p.enableAbandonedNodeEnforcement(false); Join.fullOuterJoin( p.apply("CreateLeft", Create.empty(KvCoder.of(StringUtf8Coder.of(), VarLongCoder.of()))), p.apply( "CreateRight", Create.empty(KvCoder.of(StringUtf8Coder.of(), StringUtf8Coder.of()))), -1L, null); } }