@Override public Coder<TimestampedValue<T>> getAccumulatorCoder( CoderRegistry registry, Coder<TimestampedValue<T>> inputCoder) throws CannotProvideCoderException { return NullableCoder.of(inputCoder); }
protected FileResultCoder( Coder<BoundedWindow> windowCoder, Coder<DestinationT> destinationCoder) { this.windowCoder = NullableCoder.of(windowCoder); this.destinationCoder = destinationCoder; }
@Override public NullableCoder<?> fromCloudObject(CloudObject cloudObject) { List<Coder<?>> componentList = getComponents(cloudObject); checkArgument( componentList.size() == 1, "Expected 1 component for %s, got %s", NullableCoder.class.getSimpleName(), componentList.size()); return NullableCoder.of(componentList.get(0)); }
public Coder<List> getAccumulatorCoder() { AvroCoder valueCoder = null; if (avroSchemaStr != null) { valueCoder = AvroCoder.of(new Schema.Parser().parse(avroSchemaStr)); } return (Coder<List>) (avroSchemaStr == null ? ListCoder.of(NullableCoder.of(StringUtf8Coder.of())) : ListCoder.of(NullableCoder.of(valueCoder))); } }
public Coder<List> getAccumulatorCoder() { AvroCoder valueCoder = null; if (avroSchemaStr != null) { valueCoder = AvroCoder.of(new Schema.Parser().parse(avroSchemaStr)); } return (Coder<List>) (avroSchemaStr == null ? ListCoder.of(NullableCoder.of(StringUtf8Coder.of())) : ListCoder.of(NullableCoder.of(valueCoder))); } }
@Override public Coder<T> getDefaultOutputCoder( CoderRegistry registry, Coder<TimestampedValue<T>> inputCoder) throws CannotProvideCoderException { checkState( inputCoder instanceof TimestampedValue.TimestampedValueCoder, "inputCoder must be a TimestampedValueCoder, but was %s", inputCoder); TimestampedValue.TimestampedValueCoder<T> inputTVCoder = (TimestampedValue.TimestampedValueCoder<T>) inputCoder; return NullableCoder.of(inputTVCoder.getValueCoder()); }
@Override public Coder<KV<Instant, ReadableDuration>> getStateCoder() { return KvCoder.of(NullableCoder.of(InstantCoder.of()), DurationCoder.of()); }
@Test public void testCoderIsSerializableWithWellKnownCoderType() throws Exception { CoderProperties.coderSerializable(NullableCoder.of(GlobalWindow.Coder.INSTANCE)); }
@Test public void testSubcoderRecievesEntireStream() throws Exception { NullableCoder<String> coder = NullableCoder.of(new EntireStreamExpectingCoder()); CoderProperties.coderDecodeEncodeEqualInContext(coder, Context.OUTER, null); CoderProperties.coderDecodeEncodeEqualInContext(coder, Context.OUTER, "foo"); }
@Test public void testObserverIsCheap() throws Exception { NullableCoder<Double> coder = NullableCoder.of(DoubleCoder.of()); assertTrue(coder.isRegisterByteSizeObserverCheap(5.0)); }
@Test public void testObserverIsAlwaysCheapForNullValues() throws Exception { NullableCoder<List<String>> coder = NullableCoder.of(ListCoder.of(StringUtf8Coder.of())); assertTrue(coder.isRegisterByteSizeObserverCheap(null)); }
@Test public void testEncodedSize() throws Exception { NullableCoder<Double> coder = NullableCoder.of(DoubleCoder.of()); assertEquals(1, coder.getEncodedElementByteSize(null)); assertEquals(9, coder.getEncodedElementByteSize(5.0)); }
@Override public PCollection<T> expand(PCollection<T> input) { Coder<T> inputCoder = input.getCoder(); return input .apply( "Reify Timestamps", ParDo.of( new DoFn<T, TimestampedValue<T>>() { @ProcessElement public void processElement( @Element T element, @Timestamp Instant timestamp, OutputReceiver<TimestampedValue<T>> r) { r.output(TimestampedValue.of(element, timestamp)); } })) .setCoder(TimestampedValue.TimestampedValueCoder.of(inputCoder)) .apply("Latest Value", Combine.globally(new LatestFn<>())) .setCoder(NullableCoder.of(inputCoder)); } }
@Test public void testEncodedSizeNested() throws Exception { NullableCoder<String> varLenCoder = NullableCoder.of(StringUtf8Coder.of()); assertEquals(1, varLenCoder.getEncodedElementByteSize(null)); assertEquals(6, varLenCoder.getEncodedElementByteSize("spam")); }
@Test public void testObserverIsNotCheap() throws Exception { NullableCoder<List<String>> coder = NullableCoder.of(ListCoder.of(StringUtf8Coder.of())); assertFalse(coder.isRegisterByteSizeObserverCheap(ImmutableList.of("hi", "test"))); }
@Test public void testCompoundCoder() throws Exception { SdkComponents sdkComponents = SdkComponents.create(); sdkComponents.registerEnvironment(Environments.createDockerEnvironment("java")); Coder<?> coder = VarIntCoder.of(); Coder<?> compoundCoder = NullableCoder.of(coder); String compoundCoderId = sdkComponents.registerCoder(compoundCoder); String coderId = sdkComponents.registerCoder(coder); RehydratedComponents rehydratedComponents = RehydratedComponents.forComponents(sdkComponents.toComponents()); Coder<?> rehydratedCoder = rehydratedComponents.getCoder(coderId); Coder<?> rehydratedCompoundCoder = rehydratedComponents.getCoder(compoundCoderId); assertThat(rehydratedCoder, equalTo((Coder) coder)); assertThat(rehydratedCompoundCoder, equalTo((Coder) compoundCoder)); assertThat(rehydratedComponents.getCoder(coderId), theInstance((Coder) rehydratedCoder)); assertThat( rehydratedComponents.getCoder(compoundCoderId), theInstance((Coder) rehydratedCompoundCoder)); }
@Test @Category({ValidatesRunner.class, FlattenWithHeterogeneousCoders.class}) public void testFlattenMultipleCoders() throws CannotProvideCoderException { PCollection<Long> bigEndianLongs = p.apply( "BigEndianLongs", Create.of(0L, 1L, 2L, 3L, null, 4L, 5L, null, 6L, 7L, 8L, null, 9L) .withCoder(NullableCoder.of(BigEndianLongCoder.of()))); PCollection<Long> varLongs = p.apply("VarLengthLongs", GenerateSequence.from(0).to(5)).setCoder(VarLongCoder.of()); PCollection<Long> flattened = PCollectionList.of(bigEndianLongs) .and(varLongs) .apply(Flatten.pCollections()) .setCoder(NullableCoder.of(VarLongCoder.of())); PAssert.that(flattened) .containsInAnyOrder( 0L, 0L, 1L, 1L, 2L, 3L, 2L, 4L, 5L, 3L, 6L, 7L, 4L, 8L, 9L, null, null, null); p.run(); }
@Test @Category(ValidatesRunner.class) public void testKvSwap() { PCollection<KV<String, Integer>> input = p.apply( Create.of(Arrays.asList(TABLE)) .withCoder( KvCoder.of( StringUtf8Coder.of(), NullableCoder.of(BigEndianIntegerCoder.of())))); PCollection<KV<Integer, String>> output = input.apply(KvSwap.create()); PAssert.that(output) .containsInAnyOrder( KV.of(1, "one"), KV.of(2, "two"), KV.of(3, "three"), KV.of(4, "four"), KV.of(4, "dup"), KV.of(5, "dup"), KV.of((Integer) null, "null")); p.run(); }