@Override public Coder<Integer> getOutputCoder() { return BigEndianIntegerCoder.of(); } }
@Override public QuantileState<T, ComparatorT> decode(InputStream inStream) throws CoderException, IOException { int numQuantiles = intCoder.decode(inStream); int bufferSize = intCoder.decode(inStream); T min = elementCoder.decode(inStream); T max = elementCoder.decode(inStream); List<T> unbufferedElements = elementListCoder.decode(inStream); int numBuffers = BigEndianIntegerCoder.of().decode(inStream); List<QuantileBuffer<T>> buffers = new ArrayList<>(numBuffers); for (int i = 0; i < numBuffers; i++) { buffers.add(decodeBuffer(inStream)); } return new QuantileState<>( compareFn, numQuantiles, min, max, numBuffers, bufferSize, unbufferedElements, buffers); }
@Override public void encode(QuantileState<T, ComparatorT> state, OutputStream outStream) throws CoderException, IOException { intCoder.encode(state.numQuantiles, outStream); intCoder.encode(state.bufferSize, outStream); elementCoder.encode(state.min, outStream); elementCoder.encode(state.max, outStream); elementListCoder.encode(state.unbufferedElements, outStream); BigEndianIntegerCoder.of().encode(state.buffers.size(), outStream); for (QuantileBuffer<T> buffer : state.buffers) { encodeBuffer(buffer, outStream); } }
/** * Notifies ElementByteSizeObserver about the byte size of the encoded value using this coder. */ @Override public void registerByteSizeObserver( QuantileState<T, ComparatorT> state, ElementByteSizeObserver observer) throws Exception { elementCoder.registerByteSizeObserver(state.min, observer); elementCoder.registerByteSizeObserver(state.max, observer); elementListCoder.registerByteSizeObserver(state.unbufferedElements, observer); BigEndianIntegerCoder.of().registerByteSizeObserver(state.buffers.size(), observer); for (QuantileBuffer<T> buffer : state.buffers) { observer.update(4L + 8); elementListCoder.registerByteSizeObserver(buffer.elements, observer); } }
@Override public Coder<Integer> getOutputCoder() { return BigEndianIntegerCoder.of(); }
@Override public Coder<KV<Integer, BigDecimal>> getAccumulatorCoder( CoderRegistry registry, Coder<T> inputCoder) { return KvCoder.of(BigEndianIntegerCoder.of(), BigDecimalCoder.of()); }
@Override public Coder<Top.BoundedHeap<KV<Integer, T>, SerializableComparator<KV<Integer, T>>>> getAccumulatorCoder(CoderRegistry registry, Coder<T> inputCoder) { return topCombineFn.getAccumulatorCoder( registry, KvCoder.of(BigEndianIntegerCoder.of(), inputCoder)); }
public PCollection<KV<String, Integer>> createInputTable(Pipeline p) { return p.apply( "CreateInputTable", Create.of(Arrays.asList(TABLE)) .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of()))); }
public PCollection<KV<String, Integer>> createEmptyInputTable(Pipeline p) { return p.apply( "CreateEmptyInputTable", Create.of(Arrays.asList(EMPTY_TABLE)) .withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of()))); }
@Test public void testCoderEquals() throws Exception { DelegateCoder.CodingFunction<Integer, Integer> identityFn = input -> input; Coder<Integer> varIntCoder1 = DelegateCoder.of(VarIntCoder.of(), identityFn, identityFn); Coder<Integer> varIntCoder2 = DelegateCoder.of(VarIntCoder.of(), identityFn, identityFn); Coder<Integer> bigEndianIntegerCoder = DelegateCoder.of(BigEndianIntegerCoder.of(), identityFn, identityFn); assertEquals(varIntCoder1, varIntCoder2); assertEquals(varIntCoder1.hashCode(), varIntCoder2.hashCode()); assertNotEquals(varIntCoder1, bigEndianIntegerCoder); assertNotEquals(varIntCoder1.hashCode(), bigEndianIntegerCoder.hashCode()); }
@Test public void testIntVersusStringIncompatibility() throws Exception { thrown.expect(IncompatibleCoderException.class); thrown.expectMessage("not assignable"); CoderRegistry.verifyCompatible(BigEndianIntegerCoder.of(), String.class); }
@Test public void testSourceSplit() throws Exception { CreateSource<Integer> source = CreateSource.fromIterable( ImmutableList.of(1, 2, 3, 4, 5, 6, 7, 8), BigEndianIntegerCoder.of()); PipelineOptions options = PipelineOptionsFactory.create(); List<? extends BoundedSource<Integer>> splitSources = source.split(12, options); assertThat(splitSources, hasSize(3)); SourceTestUtils.assertSourcesEqualReferenceSource(source, splitSources, options); }
private static PCollection<KV<String, Integer>> createInput( Pipeline p, List<KV<String, Integer>> table) { return p.apply( Create.of(table).withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of()))); }
@Test public void testOutputCoder() { p.enableAbandonedNodeEnforcement(false); BoundedSource<Integer> fixedCoderSource = new BigEndianIntegerSource(); assertThat( p.apply(JavaReadViaImpulse.bounded(fixedCoderSource)).getCoder(), equalTo(BigEndianIntegerCoder.of())); }
public PCollection<KV<String, Integer>> createInputTable(Pipeline p) { return p.apply( Create.of(TABLE).withCoder(KvCoder.of(StringUtf8Coder.of(), BigEndianIntegerCoder.of()))); }
@Test(expected = IllegalArgumentException.class) public void testSampleAnyNegative() { pipeline.enableAbandonedNodeEnforcement(false); pipeline.apply(Create.empty(BigEndianIntegerCoder.of())).apply(Sample.any(-10)); }
@Test public void testComponentIncompatibility() throws Exception { thrown.expect(IncompatibleCoderException.class); thrown.expectMessage("component coder is incompatible"); CoderRegistry.verifyCompatible( ListCoder.of(BigEndianIntegerCoder.of()), new TypeDescriptor<List<String>>() {}.getType()); }
base, fn, BigEndianIntegerCoder.of(), SerializableCoder.of(OffsetRange.class), max,
@Test public void testSourceSplitEmpty() throws Exception { CreateSource<Integer> source = CreateSource.fromIterable(ImmutableList.of(), BigEndianIntegerCoder.of()); PipelineOptions options = PipelineOptionsFactory.create(); List<? extends BoundedSource<Integer>> splitSources = source.split(12, options); SourceTestUtils.assertSourcesEqualReferenceSource(source, splitSources, options); }