TypeDescriptor<T> token = getTypeDescriptor();
@Override public PCollection<KV<Integer, KV<KeyT, ValueT>>> expand(PCollection<KV<KeyT, ValueT>> input) { return input .apply( "AssignTask", ParDo.of(new AssignTaskFn<KeyT, ValueT>(configView)).withSideInputs(configView)) .setTypeDescriptor( TypeDescriptors.kvs(TypeDescriptors.integers(), input.getTypeDescriptor())) .apply("GroupByTaskId", GroupByKey.create()) .apply("FlattenGroupedTasks", ParDo.of(new FlattenGroupedTasks<>())); } }
/** * Tests that when built with a concrete subclass of {@link SimpleFunction}, the type descriptor * of the output reflects its static type. */ @Test @Category(NeedsRunner.class) public void testSimpleFunctionOutputTypeDescriptor() throws Exception { PCollection<String> output = pipeline .apply(Create.of("hello")) .apply( MapElements.via( new SimpleFunction<String, String>() { @Override public String apply(String input) { return input; } })); assertThat( output.getTypeDescriptor(), equalTo((TypeDescriptor<String>) new TypeDescriptor<String>() {})); assertThat( pipeline.getCoderRegistry().getCoder(output.getTypeDescriptor()), equalTo(pipeline.getCoderRegistry().getCoder(new TypeDescriptor<String>() {}))); // Make sure the pipeline runs too pipeline.run(); }
/** * Tests that when built with a concrete subclass of {@link SimpleFunction}, the type descriptor * of the output reflects its static type. */ @Test @Category(NeedsRunner.class) public void testFlatMapFnOutputTypeDescriptor() throws Exception { PCollection<String> output = pipeline .apply(Create.of("hello")) .apply( FlatMapElements.via( new SimpleFunction<String, Set<String>>() { @Override public Set<String> apply(String input) { return ImmutableSet.copyOf(input.split("")); } })); assertThat( output.getTypeDescriptor(), equalTo((TypeDescriptor<String>) new TypeDescriptor<String>() {})); assertThat( pipeline.getCoderRegistry().getCoder(output.getTypeDescriptor()), equalTo(pipeline.getCoderRegistry().getCoder(new TypeDescriptor<String>() {}))); // Make sure the pipeline runs pipeline.run(); }
/** * Confirms that in Java 8 style, where a lambda results in a rawtype, the output type token is * not useful. If this test ever fails there may be simplifications available to us. */ @Test public void testFilterParDoOutputTypeDescriptorRawWithLambda() throws Exception { @SuppressWarnings({"unchecked", "rawtypes"}) PCollection<String> output = p.apply(Create.of("hello")).apply(Filter.by(s -> true)); thrown.expect(CannotProvideCoderException.class); p.getCoderRegistry().getCoder(output.getTypeDescriptor()); }
(Coder) registry.getCoder( out.getTypeDescriptor(), getFn().getInputTypeDescriptor(), inputCoder)); } catch (CannotProvideCoderException e) {
/** * Confirms that in Java 8 style, where a lambda results in a rawtype, the output type token is * not useful. If this test ever fails there may be simplifications available to us. */ @Test @Category(NeedsRunner.class) public void testPartitionFnOutputTypeDescriptorRaw() throws Exception { PCollectionList<String> output = pipeline.apply(Create.of("hello")).apply(Partition.of(1, (element, numPartitions) -> 0)); thrown.expect(CannotProvideCoderException.class); pipeline.getCoderRegistry().getCoder(output.get(0).getTypeDescriptor()); } }
ParDo.of( new SetupJobFn<>( externalSynchronization, configView, input.getTypeDescriptor())) .withSideInputs(configView));