@Override public Coder<Iterable<T>> getDefaultOutputCoder(CoderRegistry registry, Coder<T> inputCoder) { return IterableCoder.of(inputCoder); }
@Override public Coder<T> getOutputCoder() { return iterableCoder.getElemCoder(); }
private Iterable<T> decode(byte[] bytes) throws IOException { try (ByteArrayInputStream inputStream = new ByteArrayInputStream(bytes)) { return iterableCoder.decode(inputStream, Coder.Context.OUTER); } catch (IOException ex) { throw new RuntimeException(ex); } }
public ValuesSource(Iterable<T> values, Coder<T> coder) { this.iterableCoder = IterableCoder.of(coder); ByteArrayOutputStream bos = new ByteArrayOutputStream(); try { iterableCoder.encode(values, bos, Context.OUTER); } catch (IOException ex) { throw new RuntimeException(ex); } this.codedValues = bos.toByteArray(); }
@Override public PCollection<KV<K, Iterable<V>>> expand( PCollection<KV<K, Iterable<WindowedValue<V>>>> input) { @SuppressWarnings("unchecked") KvCoder<K, Iterable<WindowedValue<V>>> inputKvCoder = (KvCoder<K, Iterable<WindowedValue<V>>>) input.getCoder(); Coder<K> keyCoder = inputKvCoder.getKeyCoder(); Coder<Iterable<WindowedValue<V>>> inputValueCoder = inputKvCoder.getValueCoder(); IterableCoder<WindowedValue<V>> inputIterableValueCoder = (IterableCoder<WindowedValue<V>>) inputValueCoder; Coder<WindowedValue<V>> inputIterableElementCoder = inputIterableValueCoder.getElemCoder(); WindowedValueCoder<V> inputIterableWindowedValueCoder = (WindowedValueCoder<V>) inputIterableElementCoder; Coder<V> inputIterableElementValueCoder = inputIterableWindowedValueCoder.getValueCoder(); Coder<Iterable<V>> outputValueCoder = IterableCoder.of(inputIterableElementValueCoder); Coder<KV<K, Iterable<V>>> outputKvCoder = KvCoder.of(keyCoder, outputValueCoder); return PCollection.createPrimitiveOutputInternal( input.getPipeline(), windowingStrategy, input.isBounded(), outputKvCoder); } }
private byte[] encode(Iterable<T> values, IterableCoder<T> coder) { try (ByteArrayOutputStream stream = new ByteArrayOutputStream()) { coder.encode(values, stream, Coder.Context.OUTER); return stream.toByteArray(); } catch (IOException ex) { throw new RuntimeException(ex); } }
@Override @SuppressWarnings("unchecked") public void encode(CoGbkResult value, OutputStream outStream) throws CoderException, IOException { if (!schema.equals(value.getSchema())) { throw new CoderException("input schema does not match coder schema"); } if (schema.size() == 0) { return; } for (int unionTag = 0; unionTag < schema.size(); unionTag++) { tagListCoder(unionTag).encode(value.valueMap.get(unionTag), outStream); } }
@Override public Coder<Iterable<T>> getDefaultOutputCoder(CoderRegistry registry, Coder<T> inputCoder) { return IterableCoder.of(inputCoder); }
@Override public Coder<T> getDefaultOutputCoder() { return iterableCoder.getElemCoder(); }
@Override public UnboundedReader<T> createReader( PipelineOptions options, @Nullable CheckpointMark checkpointMark) { ByteArrayInputStream bis = new ByteArrayInputStream(codedValues); try { Iterable<T> values = this.iterableCoder.decode(bis, Context.OUTER); return new ValuesReader<>(values, this); } catch (IOException ex) { throw new RuntimeException(ex); } }
@Override public void encode(KafkaRecord<K, V> value, OutputStream outStream) throws IOException { stringCoder.encode(value.getTopic(), outStream); intCoder.encode(value.getPartition(), outStream); longCoder.encode(value.getOffset(), outStream); longCoder.encode(value.getTimestamp(), outStream); intCoder.encode(value.getTimestampType().ordinal(), outStream); headerCoder.encode(toIterable(value), outStream); kvCoder.encode(value.getKV(), outStream); }
@Override public IterableCoder<?> fromComponents(List<Coder<?>> components) { return IterableCoder.of(components.get(0)); } };
@Override public List<? extends Coder<?>> getComponents(IterableCoder<?> from) { return Collections.singletonList(from.getElemCoder()); }
@Override public CoGbkResult decode(InputStream inStream) throws CoderException, IOException { if (schema.size() == 0) { return new CoGbkResult(schema, ImmutableList.<Iterable<?>>of()); } List<Iterable<?>> valueMap = Lists.newArrayListWithExpectedSize(schema.size()); for (int unionTag = 0; unionTag < schema.size(); unionTag++) { valueMap.add(tagListCoder(unionTag).decode(inStream)); } return new CoGbkResult(schema, valueMap); }
@Override public void encode(KafkaRecord<K, V> value, OutputStream outStream) throws IOException { stringCoder.encode(value.getTopic(), outStream); intCoder.encode(value.getPartition(), outStream); longCoder.encode(value.getOffset(), outStream); longCoder.encode(value.getTimestamp(), outStream); intCoder.encode(value.getTimestampType().ordinal(), outStream); headerCoder.encode(toIterable(value), outStream); kvCoder.encode(value.getKV(), outStream); }
@Override public IterableCoder fromCloudObject(CloudObject object) { List<Coder<?>> components = getComponents(object); checkArgument(components.size() == 1, "Expecting 1 component, got %s", components.size()); return IterableCoder.of(components.get(0)); }
@Override public Object structuralValue(Iterable<T> value) { ArrayList<Object> result = new ArrayList<>(); for (T elem : value) { result.add(getElemCoder().structuralValue(elem)); } return result; }
@Override public KafkaRecord<K, V> decode(InputStream inStream) throws IOException { return new KafkaRecord<>( stringCoder.decode(inStream), intCoder.decode(inStream), longCoder.decode(inStream), longCoder.decode(inStream), KafkaTimestampType.forOrdinal(intCoder.decode(inStream)), (Headers) toHeaders(headerCoder.decode(inStream)), kvCoder.decode(inStream)); }
private KeyedWorkItemCoder( Coder<K> keyCoder, Coder<ElemT> elemCoder, Coder<? extends BoundedWindow> windowCoder) { this.keyCoder = keyCoder; this.elemCoder = elemCoder; this.windowCoder = windowCoder; this.timersCoder = IterableCoder.of(TimerDataCoder.of(windowCoder)); this.elemsCoder = IterableCoder.of(FullWindowedValueCoder.of(elemCoder, windowCoder)); }
/** Retrieves the {@link Coder} for the secondary key-value pairs. */ @SuppressWarnings("unchecked") private static <PrimaryKeyT, SecondaryKeyT, ValueT> KvCoder<SecondaryKeyT, ValueT> getSecondaryKeyValueCoder( Coder<KV<PrimaryKeyT, Iterable<KV<SecondaryKeyT, ValueT>>>> inputCoder) { if (!(inputCoder instanceof KvCoder)) { throw new IllegalStateException("SortValues requires its input to use KvCoder"); } @SuppressWarnings("unchecked") KvCoder<PrimaryKeyT, Iterable<KV<SecondaryKeyT, ValueT>>> kvCoder = (KvCoder<PrimaryKeyT, Iterable<KV<SecondaryKeyT, ValueT>>>) (inputCoder); if (!(kvCoder.getValueCoder() instanceof IterableCoder)) { throw new IllegalStateException( "SortValues requires the values be encoded with IterableCoder"); } IterableCoder<KV<SecondaryKeyT, ValueT>> iterableCoder = (IterableCoder<KV<SecondaryKeyT, ValueT>>) (kvCoder.getValueCoder()); if (!(iterableCoder.getElemCoder() instanceof KvCoder)) { throw new IllegalStateException( "SortValues requires the secondary key-value pairs to use KvCoder"); } return (KvCoder<SecondaryKeyT, ValueT>) (iterableCoder.getElemCoder()); }