@Override public T decode(InputStream is) throws IOException { return CoderUtils.decodeFromByteArray( innerCoder, Snappy.uncompress(ByteArrayCoder.of().decode(is))); }
@Override public void encode(T value, OutputStream os) throws IOException { ByteArrayCoder.of() .encode(Snappy.compress(CoderUtils.encodeToByteArray(innerCoder, value)), os); }
@Override public PCollection<byte[]> expand(PBegin input) { return PCollection.createPrimitiveOutputInternal( input.getPipeline(), WindowingStrategy.globalDefault(), IsBounded.BOUNDED, ByteArrayCoder.of()); } }
@Override public IsmRecord<V> decode(InputStream inStream) throws CoderException, IOException { List<Object> keyComponents = new ArrayList<>(keyComponentCoders.size()); for (Coder<?> keyCoder : keyComponentCoders) { keyComponents.add(keyCoder.decode(inStream)); } if (isMetadataKey(keyComponents)) { return IsmRecord.meta(keyComponents, ByteArrayCoder.of().decode(inStream)); } else { return IsmRecord.of(keyComponents, valueCoder.decode(inStream)); } }
@Override public void restoreKeyGroupState(int keyGroupIndex, DataInputStream in) throws Exception { checkInitCache(); Integer size = VarIntCoder.of().decode(in, Context.NESTED); for (int i = 0; i < size; i++) { byte[] idBytes = ByteArrayCoder.of().decode(in, Context.NESTED); // restore the ids which not expired. shouldOutput(keyGroupIndex, ByteBuffer.wrap(idBytes)); } }
@Test public void testRegisterByteSizeObserver() throws Exception { CoderProperties.testByteCount( ByteArrayCoder.of(), Coder.Context.OUTER, new byte[][] {{0xa, 0xb, 0xc}}); CoderProperties.testByteCount( ByteArrayCoder.of(), Coder.Context.NESTED, new byte[][] {{0xa, 0xb, 0xc}, {}, {}, {0xd, 0xe}, {}}); }
@Test public void testRegisterByteSizeObserver() throws Exception { CoderProperties.testByteCount( ByteArrayCoder.of(), Coder.Context.OUTER, new byte[][] {{0xa, 0xb, 0xc}}); CoderProperties.testByteCount( ByteArrayCoder.of(), Coder.Context.NESTED, new byte[][] {{0xa, 0xb, 0xc}, {}, {}, {0xd, 0xe}, {}}); }
@Override public OutgoingMessage decode(InputStream inStream) throws CoderException, IOException { byte[] elementBytes = ByteArrayCoder.of().decode(inStream); Map<String, String> attributes = ATTRIBUTES_CODER.decode(inStream); long timestampMsSinceEpoch = BigEndianLongCoder.of().decode(inStream); @Nullable String recordId = RECORD_ID_CODER.decode(inStream); return new OutgoingMessage(elementBytes, attributes, timestampMsSinceEpoch, recordId); } }
@Override public void encode(OutgoingMessage value, OutputStream outStream) throws CoderException, IOException { ByteArrayCoder.of().encode(value.elementBytes, outStream); ATTRIBUTES_CODER.encode(value.attributes, outStream); BigEndianLongCoder.of().encode(value.timestampMsSinceEpoch, outStream); RECORD_ID_CODER.encode(value.recordId, outStream); }
@Test public void keyedWithByteArrayKeyShouldCreateKeyedBundle() throws Exception { createKeyedBundle(ByteArrayCoder.of(), new byte[] {0, 2, 4, 99}); }
@Test public void keyedWithByteArrayKeyShouldCreateKeyedBundle() throws Exception { createKeyedBundle(ByteArrayCoder.of(), new byte[] {0, 2, 4, 99}); }
/** * Tests that {@link MutationDetectors#forValueWithCoder} does not false positive on an list of * arrays, even when some array is set to a deeply equal array that is not {@code equals}. */ @Test public void testEquivalentListOfArrays() throws Exception { List<byte[]> value = Arrays.asList(new byte[] {0x1}, new byte[] {0x2, 0x3}, new byte[] {0x4}); MutationDetector detector = MutationDetectors.forValueWithCoder(value, ListCoder.of(ByteArrayCoder.of())); value.set(0, new byte[] {0x1}); detector.verifyUnmodified(); } }
/** * Tests that {@link MutationDetectors#forValueWithCoder} does not false positive on an array, * even though it will decode is another array which Java will not say is {@code equals}. */ @Test public void testUnmodifiedArray() throws Exception { byte[] value = new byte[] {0x1, 0x2, 0x3, 0x4}; MutationDetector detector = MutationDetectors.forValueWithCoder(value, ByteArrayCoder.of()); detector.verifyUnmodified(); }
private BufferedElementCountingOutputStream createAndWriteValues( List<byte[]> values, OutputStream output) throws Exception { BufferedElementCountingOutputStream os = new BufferedElementCountingOutputStream(output, BUFFER_SIZE); for (byte[] value : values) { os.markElementStart(); ByteArrayCoder.of().encode(value, os); } return os; } }
@Test public void testTimerCoderWithInconsistentWithEqualsPayloadCoder() throws Exception { Coder<Timer<byte[]>> coder = Timer.Coder.of(ByteArrayCoder.of()); CoderProperties.coderSerializable(coder); CoderProperties.structuralValueDecodeEncodeEqual( coder, Timer.of(INSTANT, "ABC".getBytes(UTF_8))); CoderProperties.structuralValueConsistentWithEquals( coder, Timer.of(INSTANT, "ABC".getBytes(UTF_8)), Timer.of(INSTANT, "ABC".getBytes(UTF_8))); }
/** Tests that {@link MutationDetectors#forValueWithCoder} detects a mutation to a byte array. */ @Test public void testMutatingArray() throws Exception { byte[] value = new byte[] {0x1, 0x2, 0x3, 0x4}; MutationDetector detector = MutationDetectors.forValueWithCoder(value, ByteArrayCoder.of()); value[0] = 0xa; thrown.expect(IllegalMutationException.class); detector.verifyUnmodified(); }
@Test public void registerCoder() throws IOException { Coder<?> coder = KvCoder.of(StringUtf8Coder.of(), IterableCoder.of(SetCoder.of(ByteArrayCoder.of()))); String id = components.registerCoder(coder); assertThat(components.registerCoder(coder), equalTo(id)); assertThat(id, not(isEmptyOrNullString())); VarLongCoder otherCoder = VarLongCoder.of(); assertThat(components.registerCoder(otherCoder), not(equalTo(id))); components.toComponents().getCodersOrThrow(id); components.toComponents().getCodersOrThrow(components.registerCoder(otherCoder)); }
@Test public void testCoderProvidersFromStaticMethodsForParameterlessTypes() throws Exception { CoderProvider factory = CoderProviders.fromStaticMethods(String.class, StringUtf8Coder.class); assertEquals( StringUtf8Coder.of(), factory.coderFor(TypeDescriptors.strings(), Collections.emptyList())); factory = CoderProviders.fromStaticMethods(Double.class, DoubleCoder.class); assertEquals( DoubleCoder.of(), factory.coderFor(TypeDescriptors.doubles(), Collections.emptyList())); factory = CoderProviders.fromStaticMethods(byte[].class, ByteArrayCoder.class); assertEquals( ByteArrayCoder.of(), factory.coderFor(TypeDescriptor.of(byte[].class), Collections.emptyList())); }
@Before public void setup() { created = p.apply(Create.empty(ByteArrayCoder.of())); transformed = created.apply(ParDo.of(new IdentityDoFn<>())); DirectGraphVisitor visitor = new DirectGraphVisitor(); p.traverseTopologically(visitor); factory = ImmutabilityCheckingBundleFactory.create( ImmutableListBundleFactory.create(), visitor.getGraph()); }
@Test public void testStructuralKeyEquality() { MultimapView<byte[], Integer> view = InMemoryMultimapSideInputView.fromIterable( ByteArrayCoder.of(), ImmutableList.of(KV.of(new byte[] {0x00}, 0), KV.of(new byte[] {0x01}, 1))); assertEquals(view.get(new byte[] {0x00}), ImmutableList.of(0)); assertEquals(view.get(new byte[] {0x01}), ImmutableList.of(1)); assertEquals(view.get(new byte[] {0x02}), ImmutableList.of()); }