/** * Tests that when built with a concrete subclass of {@link SimpleFunction}, the type descriptor * of the output reflects its static type. */ @Test @Category(NeedsRunner.class) public void testSimpleFunctionOutputTypeDescriptor() throws Exception { PCollection<String> output = pipeline .apply(Create.of("hello")) .apply( MapElements.via( new SimpleFunction<String, String>() { @Override public String apply(String input) { return input; } })); assertThat( output.getTypeDescriptor(), equalTo((TypeDescriptor<String>) new TypeDescriptor<String>() {})); assertThat( pipeline.getCoderRegistry().getCoder(output.getTypeDescriptor()), equalTo(pipeline.getCoderRegistry().getCoder(new TypeDescriptor<String>() {}))); // Make sure the pipeline runs too pipeline.run(); }
/** * Tests that when built with a concrete subclass of {@link SimpleFunction}, the type descriptor * of the output reflects its static type. */ @Test @Category(NeedsRunner.class) public void testFlatMapFnOutputTypeDescriptor() throws Exception { PCollection<String> output = pipeline .apply(Create.of("hello")) .apply( FlatMapElements.via( new SimpleFunction<String, Set<String>>() { @Override public Set<String> apply(String input) { return ImmutableSet.copyOf(input.split("")); } })); assertThat( output.getTypeDescriptor(), equalTo((TypeDescriptor<String>) new TypeDescriptor<String>() {})); assertThat( pipeline.getCoderRegistry().getCoder(output.getTypeDescriptor()), equalTo(pipeline.getCoderRegistry().getCoder(new TypeDescriptor<String>() {}))); // Make sure the pipeline runs pipeline.run(); }
@Test public void testCreateDefaultOutputCoderUsingInference() throws Exception { Coder<Record> coder = new RecordCoder(); p.getCoderRegistry().registerCoderForClass(Record.class, coder); assertThat( p.apply(Create.of(new Record(), new Record(), new Record())).getCoder(), equalTo(coder)); }
@Test public void testCreateDefaultOutputCoderUsingTypeDescriptor() throws Exception { Coder<Record> coder = new RecordCoder(); p.getCoderRegistry().registerCoderForClass(Record.class, coder); Create.Values<Record> values = Create.of(new Record(), new Record2()).withType(new TypeDescriptor<Record>() {}); assertThat(p.apply(values).getCoder(), equalTo(coder)); }
/** * Confirms that in Java 8 style, where a lambda results in a rawtype, the output type token is * not useful. If this test ever fails there may be simplifications available to us. */ @Test public void testFilterParDoOutputTypeDescriptorRawWithLambda() throws Exception { @SuppressWarnings({"unchecked", "rawtypes"}) PCollection<String> output = p.apply(Create.of("hello")).apply(Filter.by(s -> true)); thrown.expect(CannotProvideCoderException.class); p.getCoderRegistry().getCoder(output.getTypeDescriptor()); }
/** Test that the example runs. */ @Test @Category(ValidatesRunner.class) public void testTfIdf() throws Exception { pipeline.getCoderRegistry().registerCoderForClass(URI.class, StringDelegateCoder.of(URI.class)); PCollection<KV<String, KV<URI, Double>>> wordToUriAndTfIdf = pipeline .apply( Create.of( KV.of(new URI("x"), "a b c d"), KV.of(new URI("y"), "a b c"), KV.of(new URI("z"), "a m n"))) .apply(new TfIdf.ComputeTfIdf()); PCollection<String> words = wordToUriAndTfIdf.apply(Keys.create()).apply(Distinct.create()); PAssert.that(words).containsInAnyOrder(Arrays.asList("a", "m", "n", "b", "c", "d")); pipeline.run().waitUntilFinish(); } }
@Before public void setup() throws Exception { service = new FakeBigtableService(); defaultRead = defaultRead.withBigtableService(service); defaultWrite = defaultWrite.withBigtableService(service); bigtableCoder = p.getCoderRegistry().getCoder(BIGTABLE_WRITE_TYPE); config = BigtableConfig.builder().setValidate(true).setBigtableService(service).build(); }
@Test public void testAccumulatorCombiningStateWithUnderlying() throws CannotProvideCoderException { CopyOnAccessInMemoryStateInternals<String> underlying = CopyOnAccessInMemoryStateInternals.withUnderlying(key, null); CombineFn<Long, long[], Long> sumLongFn = Sum.ofLongs(); StateNamespace namespace = new StateNamespaceForTest("foo"); CoderRegistry reg = pipeline.getCoderRegistry(); StateTag<CombiningState<Long, long[], Long>> stateTag = StateTags.combiningValue( "summer", sumLongFn.getAccumulatorCoder(reg, reg.getCoder(Long.class)), sumLongFn); GroupingState<Long, Long> underlyingValue = underlying.state(namespace, stateTag); assertThat(underlyingValue.read(), equalTo(0L)); underlyingValue.add(1L); assertThat(underlyingValue.read(), equalTo(1L)); CopyOnAccessInMemoryStateInternals<String> internals = CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying); GroupingState<Long, Long> copyOnAccessState = internals.state(namespace, stateTag); assertThat(copyOnAccessState.read(), equalTo(1L)); copyOnAccessState.add(4L); assertThat(copyOnAccessState.read(), equalTo(5L)); assertThat(underlyingValue.read(), equalTo(1L)); GroupingState<Long, Long> reReadUnderlyingValue = underlying.state(namespace, stateTag); assertThat(underlyingValue.read(), equalTo(reReadUnderlyingValue.read())); }
@Test public void testCreateTimestampedDefaultOutputCoderUsingTypeDescriptor() throws Exception { Coder<Record> coder = new RecordCoder(); p.getCoderRegistry().registerCoderForClass(Record.class, coder); Create.TimestampedValues<Record> values = Create.timestamped( TimestampedValue.of(new Record(), new Instant(0)), TimestampedValue.of(new Record2(), new Instant(0))) .withType(new TypeDescriptor<Record>() {}); assertThat(p.apply(values).getCoder(), equalTo(coder)); }
@Test public void testAccumulatorCombiningStateWithUnderlying() throws CannotProvideCoderException { CopyOnAccessInMemoryStateInternals<String> underlying = CopyOnAccessInMemoryStateInternals.withUnderlying(key, null); CombineFn<Long, long[], Long> sumLongFn = Sum.ofLongs(); StateNamespace namespace = new StateNamespaceForTest("foo"); CoderRegistry reg = pipeline.getCoderRegistry(); StateTag<CombiningState<Long, long[], Long>> stateTag = StateTags.combiningValue( "summer", sumLongFn.getAccumulatorCoder(reg, reg.getCoder(Long.class)), sumLongFn); GroupingState<Long, Long> underlyingValue = underlying.state(namespace, stateTag); assertThat(underlyingValue.read(), equalTo(0L)); underlyingValue.add(1L); assertThat(underlyingValue.read(), equalTo(1L)); CopyOnAccessInMemoryStateInternals<String> internals = CopyOnAccessInMemoryStateInternals.withUnderlying(key, underlying); GroupingState<Long, Long> copyOnAccessState = internals.state(namespace, stateTag); assertThat(copyOnAccessState.read(), equalTo(1L)); copyOnAccessState.add(4L); assertThat(copyOnAccessState.read(), equalTo(5L)); assertThat(underlyingValue.read(), equalTo(1L)); GroupingState<Long, Long> reReadUnderlyingValue = underlying.state(namespace, stateTag); assertThat(underlyingValue.read(), equalTo(reReadUnderlyingValue.read())); }
/** * Confirms that in Java 8 style, where a lambda results in a rawtype, the output type token is * not useful. If this test ever fails there may be simplifications available to us. */ @Test @Category(NeedsRunner.class) public void testPartitionFnOutputTypeDescriptorRaw() throws Exception { PCollectionList<String> output = pipeline.apply(Create.of("hello")).apply(Partition.of(1, (element, numPartitions) -> 0)); thrown.expect(CannotProvideCoderException.class); pipeline.getCoderRegistry().getCoder(output.get(0).getTypeDescriptor()); } }
combineFn.getAccumulatorCoder(pipeline.getCoderRegistry(), input.getCoder()), getAccumulatorCoder(combineProto, RehydratedComponents.forComponents(componentsProto))); assertEquals(
combineFn.getAccumulatorCoder(pipeline.getCoderRegistry(), input.getCoder()), getAccumulatorCoder(combineProto, RehydratedComponents.forComponents(componentsProto))); assertEquals(
final String stateId = "foo"; MyIntegerCoder myIntegerCoder = MyIntegerCoder.of(); pipeline.getCoderRegistry().registerCoderForClass(MyInteger.class, myIntegerCoder);
final int numKeys = 5; p.getCoderRegistry() .registerCoderProvider( CoderProviders.fromStaticMethods(BadEqualityKey.class, DeterministicKeyCoder.class));
final String stateId = "foo"; MyIntegerCoder myIntegerCoder = MyIntegerCoder.of(); pipeline.getCoderRegistry().registerCoderForClass(MyInteger.class, myIntegerCoder);
@Test @Category({ValidatesRunner.class, DataflowPortabilityApiUnsupported.class}) public void testComposedCombineNullValues() { p.getCoderRegistry() .registerCoderForClass(UserString.class, NullableCoder.of(UserStringCoder.of())); p.getCoderRegistry() .registerCoderForClass(String.class, NullableCoder.of(StringUtf8Coder.of()));
final String countStateId = "count"; Coder<MyInteger> myIntegerCoder = MyIntegerCoder.of(); pipeline.getCoderRegistry().registerCoderForClass(MyInteger.class, myIntegerCoder);
final String stateId = "foo"; Coder<MyInteger> myIntegerCoder = MyIntegerCoder.of(); pipeline.getCoderRegistry().registerCoderForClass(MyInteger.class, myIntegerCoder);
@Test @Category({ValidatesRunner.class, DataflowPortabilityApiUnsupported.class}) public void testComposedCombine() { p.getCoderRegistry().registerCoderForClass(UserString.class, UserStringCoder.of());