@Before public void setup() { stream = new ByteArrayOutputStreamWithPos(BUFFER_SIZE); }
ByteArrayOutputStreamWithPos outWithPos = new ByteArrayOutputStreamWithPos(); DataOutputViewStreamWrapper outViewWrapper = new DataOutputViewStreamWrapper(outWithPos)) {
@Override protected void writeKeyAndNamespaceSerializers(DataOutputView out) throws IOException { // the pre-versioned format only serializes the serializers, without their configuration snapshots try (ByteArrayOutputStreamWithPos stream = new ByteArrayOutputStreamWithPos()) { InstantiationUtil.serializeObject(stream, timersSnapshot.getKeySerializer()); InstantiationUtil.serializeObject(stream, timersSnapshot.getNamespaceSerializer()); out.write(stream.getBuf(), 0, stream.getPosition()); } } }
/** * Test that toString returns a substring of the buffer with range(0, position). */ @Test public void testToString() throws IOException { byte[] data = "1234567890".getBytes(ConfigConstants.DEFAULT_CHARSET); ByteArrayOutputStreamWithPos stream = new ByteArrayOutputStreamWithPos(data.length); stream.write(data); Assert.assertArrayEquals(data, stream.toString().getBytes(ConfigConstants.DEFAULT_CHARSET)); for (int i = 0; i < data.length; i++) { stream.setPosition(i); Assert.assertArrayEquals(Arrays.copyOf(data, i), stream.toString().getBytes(ConfigConstants.DEFAULT_CHARSET)); } // validate that the stored bytes are still tracked properly even when expanding array stream.setPosition(data.length + 1); Assert.assertArrayEquals(Arrays.copyOf(data, data.length + 1), stream.toString().getBytes(ConfigConstants.DEFAULT_CHARSET)); } }
/** * Verifies deserialization failure cases when reading a serializer from bytes, in the * case of a {@link InvalidClassException}. */ @Test public void testSerializerSerializationWithInvalidClass() throws Exception { TypeSerializer<?> serializer = IntSerializer.INSTANCE; byte[] serialized; try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) { TypeSerializerSerializationUtil.writeSerializer(new DataOutputViewStreamWrapper(out), serializer); serialized = out.toByteArray(); } TypeSerializer<?> deserializedSerializer; try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) { deserializedSerializer = TypeSerializerSerializationUtil.tryReadSerializer( new DataInputViewStreamWrapper(in), new ArtificialCNFExceptionThrowingClassLoader( Thread.currentThread().getContextClassLoader(), Collections.singleton(IntSerializer.class.getName())), true); } Assert.assertTrue(deserializedSerializer instanceof UnloadableDummyTypeSerializer); }
@Override public void write(DataOutputView out) throws IOException { super.write(out); if (typeSerializer instanceof UnloadableDummyTypeSerializer) { UnloadableDummyTypeSerializer<T> dummyTypeSerializer = (UnloadableDummyTypeSerializer<T>) this.typeSerializer; byte[] serializerBytes = dummyTypeSerializer.getActualBytes(); out.write(serializerBytes.length); out.write(serializerBytes); } else { // write in a way that allows the stream to recover from exceptions try (ByteArrayOutputStreamWithPos streamWithPos = new ByteArrayOutputStreamWithPos()) { InstantiationUtil.serializeObject(streamWithPos, typeSerializer); out.writeInt(streamWithPos.getPosition()); out.write(streamWithPos.getBuf(), 0, streamWithPos.getPosition()); } } }
/** * Verifies that reading and writing serializers work correctly. */ @Test public void testSerializerSerialization() throws Exception { TypeSerializer<?> serializer = IntSerializer.INSTANCE; byte[] serialized; try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) { TypeSerializerSerializationUtil.writeSerializer(new DataOutputViewStreamWrapper(out), serializer); serialized = out.toByteArray(); } TypeSerializer<?> deserializedSerializer; try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) { deserializedSerializer = TypeSerializerSerializationUtil.tryReadSerializer( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader()); } Assert.assertEquals(serializer, deserializedSerializer); }
/** * Verifies deserialization failure cases when reading a serializer from bytes, in the * case of a {@link ClassNotFoundException}. */ @Test public void testSerializerSerializationWithClassNotFound() throws Exception { TypeSerializer<?> serializer = IntSerializer.INSTANCE; byte[] serialized; try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) { TypeSerializerSerializationUtil.writeSerializer(new DataOutputViewStreamWrapper(out), serializer); serialized = out.toByteArray(); } TypeSerializer<?> deserializedSerializer; try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) { deserializedSerializer = TypeSerializerSerializationUtil.tryReadSerializer( new DataInputViewStreamWrapper(in), new ArtificialCNFExceptionThrowingClassLoader( Thread.currentThread().getContextClassLoader(), Collections.singleton(IntSerializer.class.getName())), true); } Assert.assertTrue(deserializedSerializer instanceof UnloadableDummyTypeSerializer); Assert.assertArrayEquals( InstantiationUtil.serializeObject(serializer), ((UnloadableDummyTypeSerializer<?>) deserializedSerializer).getActualBytes()); }
ByteArrayOutputStreamWithPos bufferWithPos = new ByteArrayOutputStreamWithPos(); DataOutputViewStreamWrapper bufferWrapper = new DataOutputViewStreamWrapper(bufferWithPos)) {
@Test public void testReadNonVersioned() throws IOException { int preVersionedPayload = 563; TestNonVersionedReadableWritable nonVersionedReadableWritable = new TestNonVersionedReadableWritable(preVersionedPayload); byte[] serialized; try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) { nonVersionedReadableWritable.write(new DataOutputViewStreamWrapper(out)); serialized = out.toByteArray(); } TestPostVersionedReadableWritable restoredVersionedReadableWritable = new TestPostVersionedReadableWritable(); try(ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) { restoredVersionedReadableWritable.read(in); } Assert.assertEquals(String.valueOf(preVersionedPayload), restoredVersionedReadableWritable.getData()); }
@Test public void testReadVersioned() throws IOException { String payload = "test-data"; TestPostVersionedReadableWritable versionedReadableWritable = new TestPostVersionedReadableWritable(payload); byte[] serialized; try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) { versionedReadableWritable.write(new DataOutputViewStreamWrapper(out)); serialized = out.toByteArray(); } TestPostVersionedReadableWritable restoredVersionedReadableWritable = new TestPostVersionedReadableWritable(); try(ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) { restoredVersionedReadableWritable.read(in); } Assert.assertEquals(payload, restoredVersionedReadableWritable.getData()); }
@Test public void testReadSameVersion() throws Exception { String payload = "test"; TestWriteable testWriteable = new TestWriteable(1, payload); byte[] serialized; try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) { testWriteable.write(new DataOutputViewStreamWrapper(out)); serialized = out.toByteArray(); } testWriteable = new TestWriteable(1); try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) { testWriteable.read(new DataInputViewStreamWrapper(in)); } Assert.assertEquals(payload, testWriteable.getData()); }
@Test public void testReadCompatibleVersion() throws Exception { String payload = "test"; TestWriteable testWriteable = new TestWriteable(1, payload); byte[] serialized; try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) { testWriteable.write(new DataOutputViewStreamWrapper(out)); serialized = out.toByteArray(); } testWriteable = new TestWriteable(2) { @Override public int[] getCompatibleVersions() { return new int[] {1, 2}; } }; try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) { testWriteable.read(new DataInputViewStreamWrapper(in)); } Assert.assertEquals(payload, testWriteable.getData()); }
@Test public void testReadMismatchVersion() throws Exception { String payload = "test"; TestWriteable testWriteable = new TestWriteable(1, payload); byte[] serialized; try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) { testWriteable.write(new DataOutputViewStreamWrapper(out)); serialized = out.toByteArray(); } testWriteable = new TestWriteable(2); try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) { testWriteable.read(new DataInputViewStreamWrapper(in)); Assert.fail("Version mismatch expected."); } catch (VersionMismatchException ignored) { } Assert.assertEquals(null, testWriteable.getData()); }
ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos(64);
@Override public byte[] deserialize(DataInputView source) throws IOException { ByteArrayOutputStreamWithPos output = new ByteArrayOutputStreamWithPos(); byte[] buffer = new byte[1024]; int numBytes; source.skipBytesToRead(1); while ((numBytes = source.read(buffer)) != -1) { output.write(buffer, 0, numBytes); } return output.toByteArray(); }
@Override protected void writeKeyAndNamespaceSerializers(DataOutputView out) throws IOException { // the pre-versioned format only serializes the serializers, without their configuration snapshots try (ByteArrayOutputStreamWithPos stream = new ByteArrayOutputStreamWithPos()) { InstantiationUtil.serializeObject(stream, timersSnapshot.getKeySerializer()); InstantiationUtil.serializeObject(stream, timersSnapshot.getNamespaceSerializer()); out.write(stream.getBuf(), 0, stream.getPosition()); } } }
@Override protected void writeKeyAndNamespaceSerializers(DataOutputView out) throws IOException { // the pre-versioned format only serializes the serializers, without their configuration snapshots try (ByteArrayOutputStreamWithPos stream = new ByteArrayOutputStreamWithPos()) { InstantiationUtil.serializeObject(stream, timersSnapshot.getKeySerializer()); InstantiationUtil.serializeObject(stream, timersSnapshot.getNamespaceSerializer()); out.write(stream.getBuf(), 0, stream.getPosition()); } } }
@Override public int compare(T left, T right) { ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos(); DataOutputViewStreamWrapper outputView = new DataOutputViewStreamWrapper(outputStream); try { serializer.serialize(left, outputView); byte[] leftBytes = outputStream.toByteArray(); outputStream.reset(); serializer.serialize(right, outputView); byte[] rightBytes = outputStream.toByteArray(); return compareBytes(leftBytes, rightBytes); } catch (IOException e) { throw new SerializationException(e); } }
@Override public KeyedStateRow map(Tuple2<K, V> t) throws Exception { int keyGroup = KeyGroupRangeAssignment.assignToKeyGroup(t.f0, maxParallelism); ByteArrayOutputStreamWithPos os = new ByteArrayOutputStreamWithPos(); DataOutputViewStreamWrapper ov = new DataOutputViewStreamWrapper(os); RocksDBUtils.writeKeyGroup(keyGroup, keygroupPrefixBytes, ov); RocksDBUtils.writeKey(t.f0, keySerializer, os, ov, false); RocksDBUtils.writeNameSpace(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, os, ov, false); os.close(); return new KeyedStateRow(stateName, os.toByteArray(), InstantiationUtil.serializeToByteArray(valueSerializer, t.f1)); } }