@Override public Coder<KV<Instant, ReadableDuration>> getStateCoder() { return KvCoder.of(InstantCoder.of(), DurationCoder.of()); }
@Override public TimestampedValue<T> decode(InputStream inStream) throws IOException { T value = valueCoder.decode(inStream); Instant timestamp = InstantCoder.of().decode(inStream); return TimestampedValue.of(value, timestamp); }
@Override public Timer<T> decode(InputStream inStream) throws CoderException, IOException { Instant instant = InstantCoder.of().decode(inStream); T value = payloadCoder.decode(inStream); return Timer.of(instant, value); }
@Override public Coder<KV<Instant, ReadableDuration>> getStateCoder() { return KvCoder.of(NullableCoder.of(InstantCoder.of()), DurationCoder.of()); }
public FlinkWatermarkHoldState( KeyedStateBackend<ByteBuffer> flinkStateBackend, Map<String, Instant> watermarkHolds, String stateId, StateNamespace namespace, TimestampCombiner timestampCombiner) { this.stateId = stateId; this.timestampCombiner = timestampCombiner; this.namespace = namespace; this.flinkStateBackend = flinkStateBackend; this.watermarkHolds = watermarkHolds; flinkStateDescriptor = new ValueStateDescriptor<>(stateId, new CoderTypeSerializer<>(InstantCoder.of())); }
public FlinkWatermarkHoldState( KeyedStateBackend<ByteBuffer> flinkStateBackend, Map<String, Instant> watermarkHolds, String stateId, StateNamespace namespace, TimestampCombiner timestampCombiner) { this.stateId = stateId; this.timestampCombiner = timestampCombiner; this.namespace = namespace; this.flinkStateBackend = flinkStateBackend; this.watermarkHolds = watermarkHolds; flinkStateDescriptor = new ValueStateDescriptor<>(stateId, new CoderTypeSerializer<>(InstantCoder.of())); }
@Override public void encode(TimestampedValue<T> windowedElem, OutputStream outStream) throws IOException { valueCoder.encode(windowedElem.getValue(), outStream); InstantCoder.of().encode(windowedElem.getTimestamp(), outStream); }
@Override public void registerByteSizeObserver(Timer<T> value, ElementByteSizeObserver observer) throws Exception { InstantCoder.of().registerByteSizeObserver(value.getTimestamp(), observer); payloadCoder.registerByteSizeObserver(value.getPayload(), observer); } }
@Override public void encode(Timer<T> timer, OutputStream outStream) throws CoderException, IOException { InstantCoder.of().encode(timer.getTimestamp(), outStream); payloadCoder.encode(timer.getPayload(), outStream); }
public FlinkWatermarkHoldState( KeyedStateBackend<ByteBuffer> flinkStateBackend, FlinkStateInternals<K> flinkStateInternals, StateTag<WatermarkHoldState> address, StateNamespace namespace, TimestampCombiner timestampCombiner) { this.address = address; this.timestampCombiner = timestampCombiner; this.namespace = namespace; this.flinkStateBackend = flinkStateBackend; this.flinkStateInternals = flinkStateInternals; flinkStateDescriptor = new ValueStateDescriptor<>( address.getId(), new CoderTypeSerializer<>(InstantCoder.of())); }
@Override public ValueInSingleWindow<T> decode(InputStream inStream, Context context) throws IOException { Instant timestamp = InstantCoder.of().decode(inStream); BoundedWindow window = windowCoder.decode(inStream); PaneInfo pane = PaneInfo.PaneInfoCoder.INSTANCE.decode(inStream); T value = valueCoder.decode(inStream, context); return new AutoValue_ValueInSingleWindow<>(value, timestamp, window, pane); }
@Override public WindowedValue<T> decode(InputStream inStream, Context context) throws CoderException, IOException { Instant timestamp = InstantCoder.of().decode(inStream); Collection<? extends BoundedWindow> windows = windowsCoder.decode(inStream); PaneInfo pane = PaneInfoCoder.INSTANCE.decode(inStream); T value = valueCoder.decode(inStream, context); // Because there are some remaining (incorrect) uses of WindowedValue with no windows, // we call this deprecated no-validation path when decoding return WindowedValue.createWithoutValidation(value, timestamp, windows, pane); }
@Override public void encode(WindowedValue<T> windowedElem, OutputStream outStream, Context context) throws CoderException, IOException { InstantCoder.of().encode(windowedElem.getTimestamp(), outStream); windowsCoder.encode(windowedElem.getWindows(), outStream); PaneInfoCoder.INSTANCE.encode(windowedElem.getPane(), outStream); valueCoder.encode(windowedElem.getValue(), outStream, context); }
@Override public void registerByteSizeObserver(WindowedValue<T> value, ElementByteSizeObserver observer) throws Exception { InstantCoder.of().registerByteSizeObserver(value.getTimestamp(), observer); windowsCoder.registerByteSizeObserver(value.getWindows(), observer); PaneInfoCoder.INSTANCE.registerByteSizeObserver(value.getPane(), observer); valueCoder.registerByteSizeObserver(value.getValue(), observer); }
@Override public void encode(ValueInSingleWindow<T> windowedElem, OutputStream outStream, Context context) throws IOException { InstantCoder.of().encode(windowedElem.getTimestamp(), outStream); windowCoder.encode(windowedElem.getWindow(), outStream); PaneInfo.PaneInfoCoder.INSTANCE.encode(windowedElem.getPane(), outStream); valueCoder.encode(windowedElem.getValue(), outStream, context); }
@Override public PCollection<KV<K, Iterable<KV<Instant, WindowedValue<KV<K, V>>>>>> expand( PCollection<KV<K, V>> input) { WindowingStrategy<?, ?> inputWindowingStrategy = input.getWindowingStrategy(); // A KvCoder is required since this goes through GBK. Further, WindowedValueCoder // is not registered by default, so we explicitly set the relevant coders. checkState( input.getCoder() instanceof KvCoder, "Input to a %s using state requires a %s, but the coder was %s", ParDo.class.getSimpleName(), KvCoder.class.getSimpleName(), input.getCoder()); KvCoder<K, V> kvCoder = (KvCoder<K, V>) input.getCoder(); Coder<K> keyCoder = kvCoder.getKeyCoder(); Coder<? extends BoundedWindow> windowCoder = inputWindowingStrategy.getWindowFn().windowCoder(); return input // Stash the original timestamps, etc, for when it is fed to the user's DoFn .apply("ReifyWindows", ParDo.of(new ReifyWindowedValueFn<>())) .setCoder( KvCoder.of( keyCoder, KvCoder.of(InstantCoder.of(), WindowedValue.getFullCoder(kvCoder, windowCoder)))) // Group by key and sort by timestamp, dropping windows as they are reified .apply("PartitionKeys", new GroupByKeyAndSortValuesOnly<>()) // The GBKO sets the windowing strategy to the global default .setWindowingStrategyInternal(inputWindowingStrategy); } }
@Override public PDone expand(PCollection<KV<K, V>> in) { // Make sure that a window has been applied. in = ofDefaultWindow(in); // Add an artificial GroupByKey to collect the window results together. PCollection<KV<Instant, KV<K, V>>> pc2 = in.apply("GroupToOneShard", ParDo.of(new GroupToOneShard<KV<K, V>>())).setCoder( KvCoder.of(InstantCoder.of(), in.getCoder())); PCollection<KV<Instant, Iterable<KV<K, V>>>> pc3 = pc2.apply(GroupByKey.<Instant, KV<K, V>> create()); pc3.apply("UnboundedWrite", ParDo.of(new UnboundedWriteToFile<K, V>(sink))); return PDone.in(in.getPipeline()); } }
@Override public PDone expand(PCollection<KV<K, V>> in) { // Make sure that a window has been applied. in = ofDefaultWindow(in); // Add an artificial GroupByKey to collect the window results together. PCollection<KV<Instant, KV<K, V>>> pc2 = in.apply("GroupToOneShard", ParDo.of(new GroupToOneShard<KV<K, V>>())).setCoder( KvCoder.of(InstantCoder.of(), in.getCoder())); PCollection<KV<Instant, Iterable<KV<K, V>>>> pc3 = pc2.apply(GroupByKey.<Instant, KV<K, V>> create()); pc3.apply("UnboundedWrite", ParDo.of(new UnboundedWriteToFile<K, V>(sink))); return PDone.in(in.getPipeline()); } }
Instant dayEnd = Instant.parse("2015-04-02T00:00:00Z"); Coder<Instant> instantCoder = InstantCoder.of(); byte[] encodedStart = CoderUtils.encodeToByteArray(instantCoder, start); byte[] encodedMinuteEnd = CoderUtils.encodeToByteArray(instantCoder, minuteEnd);
@Test public void testUpdatesWatermark() throws Exception { DoFn<Instant, String> fn = new WatermarkUpdateFn(); Instant base = Instant.now(); ProcessFnTester<Instant, String, OffsetRange, Long> tester = new ProcessFnTester<>( base, fn, InstantCoder.of(), SerializableCoder.of(OffsetRange.class), 3, MAX_BUNDLE_DURATION); tester.startElement(base, new OffsetRange(0, 8)); assertThat(tester.takeOutputElements(), hasItems("0", "1", "2")); assertEquals(base.plus(Duration.standardSeconds(2)), tester.getWatermarkHold()); assertTrue(tester.advanceProcessingTimeBy(Duration.standardSeconds(1))); assertThat(tester.takeOutputElements(), hasItems("3", "4", "5")); assertEquals(base.plus(Duration.standardSeconds(5)), tester.getWatermarkHold()); assertTrue(tester.advanceProcessingTimeBy(Duration.standardSeconds(1))); assertThat(tester.takeOutputElements(), hasItems("6", "7")); assertEquals(null, tester.getWatermarkHold()); }