@Override public ByteBuffer getKey(WindowedValue<ValueWithRecordId<T>> value) throws Exception { return ByteBuffer.wrap(value.getValue().getId()); }
@Override public ByteBuffer getKey(WindowedValue<ValueWithRecordId<T>> value) throws Exception { return ByteBuffer.wrap(value.getValue().getId()); }
@Override public ByteBuffer getKey(WindowedValue<ValueWithRecordId<T>> value) throws Exception { return ByteBuffer.wrap(value.getValue().getId()); }
@Override public ByteBuffer getKey(WindowedValue<ValueWithRecordId<String>> value) throws Exception { return ByteBuffer.wrap(value.getValue().getId()); } }, TypeInformation.of(ByteBuffer.class));
private KeyedOneInputStreamOperatorTestHarness< ByteBuffer, WindowedValue<ValueWithRecordId<String>>, WindowedValue<String>> getDebupingHarness() throws Exception { DedupingOperator<String> operator = new DedupingOperator<>(); return new KeyedOneInputStreamOperatorTestHarness<>( operator, value -> ByteBuffer.wrap(value.getValue().getId()), TypeInformation.of(ByteBuffer.class)); } }
private KeyedOneInputStreamOperatorTestHarness< ByteBuffer, WindowedValue<ValueWithRecordId<String>>, WindowedValue<String>> getDebupingHarness() throws Exception { DedupingOperator<String> operator = new DedupingOperator<>(); return new KeyedOneInputStreamOperatorTestHarness<>( operator, value -> ByteBuffer.wrap(value.getValue().getId()), TypeInformation.of(ByteBuffer.class)); } }
@Override public PCollection<T> expand(PCollection<ValueWithRecordId<T>> input) { return input .apply( WithKeys.of( (ValueWithRecordId<T> value) -> Arrays.hashCode(value.getId()) % NUM_RESHARD_KEYS) .withKeyType(TypeDescriptors.integers())) // Reshuffle will dedup based on ids in ValueWithRecordId by passing the data through // WindmillSink. .apply(Reshuffle.of()) .apply( "StripIds", ParDo.of( new DoFn<KV<Integer, ValueWithRecordId<T>>, T>() { @ProcessElement public void processElement(ProcessContext c) { c.output(c.element().getValue().getValue()); } })); } }