@Override public Coder<KV<Instant, ReadableDuration>> getStateCoder() { return KvCoder.of(InstantCoder.of(), DurationCoder.of()); }
@Override public void registerByteSizeObserver(Timer<T> value, ElementByteSizeObserver observer) throws Exception { InstantCoder.of().registerByteSizeObserver(value.getTimestamp(), observer); payloadCoder.registerByteSizeObserver(value.getPayload(), observer); } }
@Override public void encode(KinesisRecord value, OutputStream outStream) throws IOException { BYTE_ARRAY_CODER.encode(value.getData().array(), outStream); STRING_CODER.encode(value.getSequenceNumber(), outStream); STRING_CODER.encode(value.getPartitionKey(), outStream); INSTANT_CODER.encode(value.getApproximateArrivalTimestamp(), outStream); VAR_LONG_CODER.encode(value.getSubSequenceNumber(), outStream); INSTANT_CODER.encode(value.getReadTime(), outStream); STRING_CODER.encode(value.getStreamName(), outStream); STRING_CODER.encode(value.getShardId(), outStream); }
@Override public KinesisRecord decode(InputStream inStream) throws IOException { ByteBuffer data = ByteBuffer.wrap(BYTE_ARRAY_CODER.decode(inStream)); String sequenceNumber = STRING_CODER.decode(inStream); String partitionKey = STRING_CODER.decode(inStream); Instant approximateArrivalTimestamp = INSTANT_CODER.decode(inStream); long subSequenceNumber = VAR_LONG_CODER.decode(inStream); Instant readTimestamp = INSTANT_CODER.decode(inStream); String streamName = STRING_CODER.decode(inStream); String shardId = STRING_CODER.decode(inStream); return new KinesisRecord( data, sequenceNumber, subSequenceNumber, partitionKey, approximateArrivalTimestamp, readTimestamp, streamName, shardId); } }
@Override public void registerByteSizeObserver(WindowedValue<T> value, ElementByteSizeObserver observer) throws Exception { InstantCoder.of().registerByteSizeObserver(value.getTimestamp(), observer); windowsCoder.registerByteSizeObserver(value.getWindows(), observer); PaneInfoCoder.INSTANCE.registerByteSizeObserver(value.getPane(), observer); valueCoder.registerByteSizeObserver(value.getValue(), observer); }
@Override public void encode(TimerData timer, OutputStream outStream) throws CoderException, IOException { STRING_CODER.encode(timer.getTimerId(), outStream); STRING_CODER.encode(timer.getNamespace().stringKey(), outStream); INSTANT_CODER.encode(timer.getTimestamp(), outStream); STRING_CODER.encode(timer.getDomain().name(), outStream); }
@Override public TimerData decode(InputStream inStream) throws CoderException, IOException { String timerId = STRING_CODER.decode(inStream); StateNamespace namespace = StateNamespaces.fromString(STRING_CODER.decode(inStream), windowCoder); Instant timestamp = INSTANT_CODER.decode(inStream); TimeDomain domain = TimeDomain.valueOf(STRING_CODER.decode(inStream)); return TimerData.of(timerId, namespace, timestamp, domain); }
@Override public Coder<KV<Instant, ReadableDuration>> getStateCoder() { return KvCoder.of(NullableCoder.of(InstantCoder.of()), DurationCoder.of()); }
public FlinkWatermarkHoldState( KeyedStateBackend<ByteBuffer> flinkStateBackend, Map<String, Instant> watermarkHolds, String stateId, StateNamespace namespace, TimestampCombiner timestampCombiner) { this.stateId = stateId; this.timestampCombiner = timestampCombiner; this.namespace = namespace; this.flinkStateBackend = flinkStateBackend; this.watermarkHolds = watermarkHolds; flinkStateDescriptor = new ValueStateDescriptor<>(stateId, new CoderTypeSerializer<>(InstantCoder.of())); }
@Override public void encode(WindowedValue<T> windowedElem, OutputStream outStream, Context context) throws CoderException, IOException { InstantCoder.of().encode(windowedElem.getTimestamp(), outStream); windowsCoder.encode(windowedElem.getWindows(), outStream); PaneInfoCoder.INSTANCE.encode(windowedElem.getPane(), outStream); valueCoder.encode(windowedElem.getValue(), outStream, context); }
@Override public ValueInSingleWindow<T> decode(InputStream inStream, Context context) throws IOException { Instant timestamp = InstantCoder.of().decode(inStream); BoundedWindow window = windowCoder.decode(inStream); PaneInfo pane = PaneInfo.PaneInfoCoder.INSTANCE.decode(inStream); T value = valueCoder.decode(inStream, context); return new AutoValue_ValueInSingleWindow<>(value, timestamp, window, pane); }
public FlinkWatermarkHoldState( KeyedStateBackend<ByteBuffer> flinkStateBackend, Map<String, Instant> watermarkHolds, String stateId, StateNamespace namespace, TimestampCombiner timestampCombiner) { this.stateId = stateId; this.timestampCombiner = timestampCombiner; this.namespace = namespace; this.flinkStateBackend = flinkStateBackend; this.watermarkHolds = watermarkHolds; flinkStateDescriptor = new ValueStateDescriptor<>(stateId, new CoderTypeSerializer<>(InstantCoder.of())); }
@Override public void encode(ValueInSingleWindow<T> windowedElem, OutputStream outStream, Context context) throws IOException { InstantCoder.of().encode(windowedElem.getTimestamp(), outStream); windowCoder.encode(windowedElem.getWindow(), outStream); PaneInfo.PaneInfoCoder.INSTANCE.encode(windowedElem.getPane(), outStream); valueCoder.encode(windowedElem.getValue(), outStream, context); }
@Override public WindowedValue<T> decode(InputStream inStream, Context context) throws CoderException, IOException { Instant timestamp = InstantCoder.of().decode(inStream); Collection<? extends BoundedWindow> windows = windowsCoder.decode(inStream); PaneInfo pane = PaneInfoCoder.INSTANCE.decode(inStream); T value = valueCoder.decode(inStream, context); // Because there are some remaining (incorrect) uses of WindowedValue with no windows, // we call this deprecated no-validation path when decoding return WindowedValue.createWithoutValidation(value, timestamp, windows, pane); }
public FlinkWatermarkHoldState( KeyedStateBackend<ByteBuffer> flinkStateBackend, FlinkStateInternals<K> flinkStateInternals, StateTag<WatermarkHoldState> address, StateNamespace namespace, TimestampCombiner timestampCombiner) { this.address = address; this.timestampCombiner = timestampCombiner; this.namespace = namespace; this.flinkStateBackend = flinkStateBackend; this.flinkStateInternals = flinkStateInternals; flinkStateDescriptor = new ValueStateDescriptor<>( address.getId(), new CoderTypeSerializer<>(InstantCoder.of())); }
@Override public PCollection<KV<K, Iterable<KV<Instant, WindowedValue<KV<K, V>>>>>> expand( PCollection<KV<K, V>> input) { WindowingStrategy<?, ?> inputWindowingStrategy = input.getWindowingStrategy(); // A KvCoder is required since this goes through GBK. Further, WindowedValueCoder // is not registered by default, so we explicitly set the relevant coders. checkState( input.getCoder() instanceof KvCoder, "Input to a %s using state requires a %s, but the coder was %s", ParDo.class.getSimpleName(), KvCoder.class.getSimpleName(), input.getCoder()); KvCoder<K, V> kvCoder = (KvCoder<K, V>) input.getCoder(); Coder<K> keyCoder = kvCoder.getKeyCoder(); Coder<? extends BoundedWindow> windowCoder = inputWindowingStrategy.getWindowFn().windowCoder(); return input // Stash the original timestamps, etc, for when it is fed to the user's DoFn .apply("ReifyWindows", ParDo.of(new ReifyWindowedValueFn<>())) .setCoder( KvCoder.of( keyCoder, KvCoder.of(InstantCoder.of(), WindowedValue.getFullCoder(kvCoder, windowCoder)))) // Group by key and sort by timestamp, dropping windows as they are reified .apply("PartitionKeys", new GroupByKeyAndSortValuesOnly<>()) // The GBKO sets the windowing strategy to the global default .setWindowingStrategyInternal(inputWindowingStrategy); } }