@Override protected void writeKeyAndNamespaceSerializers(DataOutputView out) throws IOException { // the pre-versioned format only serializes the serializers, without their configuration snapshots try (ByteArrayOutputStreamWithPos stream = new ByteArrayOutputStreamWithPos()) { InstantiationUtil.serializeObject(stream, timersSnapshot.getKeySerializer()); InstantiationUtil.serializeObject(stream, timersSnapshot.getNamespaceSerializer()); out.write(stream.getBuf(), 0, stream.getPosition()); } } }
/** * Verifies that reading and writing serializers work correctly. */ @Test public void testSerializerSerialization() throws Exception { TypeSerializer<?> serializer = IntSerializer.INSTANCE; byte[] serialized; try (ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos()) { TypeSerializerSerializationUtil.writeSerializer(new DataOutputViewStreamWrapper(out), serializer); serialized = out.toByteArray(); } TypeSerializer<?> deserializedSerializer; try (ByteArrayInputStreamWithPos in = new ByteArrayInputStreamWithPos(serialized)) { deserializedSerializer = TypeSerializerSerializationUtil.tryReadSerializer( new DataInputViewStreamWrapper(in), Thread.currentThread().getContextClassLoader()); } Assert.assertEquals(serializer, deserializedSerializer); }
ByteArrayOutputStreamWithPos out = new ByteArrayOutputStreamWithPos(64); out.reset(); int size = i % 4; int end = prev + size; prev = end + 1; for (int kg : offsets.getKeyGroupRange()) { offsets.setKeyGroupOffset(kg, out.getPosition()); dov.writeInt(kg); ++writtenKeyGroups; new KeyGroupsStateHandle(offsets, new ByteStateHandleCloseChecking("kg-" + i, out.toByteArray())); out.reset(); DataOutputView dov = new DataOutputViewStreamWrapper(out); LongArrayList offsets = new LongArrayList(size); for (int s = 0; s < size; ++s) { offsets.add(out.getPosition()); int val = i * NUM_HANDLES + s; dov.writeInt(val); new OperatorStateHandle.StateMetaInfo(offsets.toArray(), OperatorStateHandle.Mode.SPLIT_DISTRIBUTE)); OperatorStateHandle operatorStateHandle = new OperatorStreamStateHandle(offsetsMap, new ByteStateHandleCloseChecking("os-" + i, out.toByteArray())); operatorStateHandles.add(operatorStateHandle);
/** * Test setting position larger than buffer size. */ @Test public void testSetPositionLargerThanBufferSize() throws Exception { // fully fill the buffer stream.write(new byte[BUFFER_SIZE]); Assert.assertEquals(BUFFER_SIZE, stream.getBuf().length); // expand the buffer by setting position beyond the buffer length stream.setPosition(BUFFER_SIZE + 1); Assert.assertEquals(BUFFER_SIZE * 2, stream.getBuf().length); Assert.assertEquals(BUFFER_SIZE + 1, stream.getPosition()); }
/** * Test that toString returns a substring of the buffer with range(0, position). */ @Test public void testToString() throws IOException { byte[] data = "1234567890".getBytes(ConfigConstants.DEFAULT_CHARSET); ByteArrayOutputStreamWithPos stream = new ByteArrayOutputStreamWithPos(data.length); stream.write(data); Assert.assertArrayEquals(data, stream.toString().getBytes(ConfigConstants.DEFAULT_CHARSET)); for (int i = 0; i < data.length; i++) { stream.setPosition(i); Assert.assertArrayEquals(Arrays.copyOf(data, i), stream.toString().getBytes(ConfigConstants.DEFAULT_CHARSET)); } // validate that the stored bytes are still tracked properly even when expanding array stream.setPosition(data.length + 1); Assert.assertArrayEquals(Arrays.copyOf(data, data.length + 1), stream.toString().getBytes(ConfigConstants.DEFAULT_CHARSET)); } }
/** * Decode a byte array, byte-for-byte copy. */ public byte[] decodeByteArray(DataInputView source, Order ord) throws IOException { if (reuseBuffer == null) { reuseBuffer = new byte[BUFFER_SIZE]; } if (reuseBaos == null) { reuseBaos = new ByteArrayOutputStreamWithPos(); } reuseBaos.reset(); int len; while ((len = source.read(reuseBuffer)) != -1) { reuseBaos.write(reuseBuffer, 0, len); } if (ord == Order.DESCENDING) { // DESCENDING ordered ByteArray requires a termination bit to preserve // sort-order semantics of empty values. reuseBaos.setPosition(reuseBaos.size() - 1); } return ord.apply(reuseBaos.toByteArray()); }
@Override public long getPos() throws IOException { return os.getPosition(); }
@Override public void add(V value) throws IOException { try { writeCurrentKeyWithGroupAndNamespace(); byte[] key = keySerializationStream.toByteArray(); keySerializationStream.reset(); DataOutputViewStreamWrapper out = new DataOutputViewStreamWrapper(keySerializationStream); valueSerializer.serialize(value, out); backend.db.merge(columnFamily, writeOptions, key, keySerializationStream.toByteArray()); } catch (Exception e) { throw new RuntimeException("Error while adding data to RocksDB", e); } }
/** * Decode a bigint value. */ public BigInteger decodeBigInteger(DataInputView source, Order ord) throws IOException { // skip the first 4 bytes which is only used for sort source.skipBytes(4); if (reuseBuffer == null) { reuseBuffer = new byte[BUFFER_SIZE]; } if (reuseBaos == null) { reuseBaos = new ByteArrayOutputStreamWithPos(); } reuseBaos.reset(); int len; while ((len = source.read(reuseBuffer)) != -1) { reuseBaos.write(reuseBuffer, 0, len); } return new BigInteger(ord.apply(reuseBaos.toByteArray())); }
public static <K> byte[] getSerializedKeyForKeyedListState( ByteArrayOutputStreamWithPos outputStream, DataOutputView outputView, K key, TypeSerializer<K> keySerializer, int group, byte[] stateNameByte) throws IOException { serializeSingleKey(outputView, key, keySerializer, group, stateNameByte); return outputStream.toByteArray(); }
/** * Closes the stream and returns the byte array containing the stream's data. * @return The byte array containing the stream's data. * @throws IOException Thrown if the size of the data exceeds the maximal */ public byte[] closeAndGetBytes() throws IOException { if (closed.compareAndSet(false, true)) { checkSize(os.size(), maxSize); byte[] bytes = os.toByteArray(); closeInternal(); return bytes; } else { throw new IOException("stream has already been closed"); } }
@Override public int compare(T left, T right) { ByteArrayOutputStreamWithPos outputStream = new ByteArrayOutputStreamWithPos(); DataOutputViewStreamWrapper outputView = new DataOutputViewStreamWrapper(outputStream); try { serializer.serialize(left, outputView); byte[] leftBytes = outputStream.toByteArray(); outputStream.reset(); serializer.serialize(right, outputView); byte[] rightBytes = outputStream.toByteArray(); return compareBytes(leftBytes, rightBytes); } catch (IOException e) { throw new SerializationException(e); } }
@Override public void write(byte[] b, int off, int len) throws IOException { os.write(b, off, len); isEmpty = false; }
@Override public void close() { if (!closed) { closed = true; os.reset(); } } }
@Override public byte[] deserialize(DataInputView source) throws IOException { ByteArrayOutputStreamWithPos output = new ByteArrayOutputStreamWithPos(); byte[] buffer = new byte[1024]; int numBytes; source.skipBytesToRead(1); while ((numBytes = source.read(buffer)) != -1) { output.write(buffer, 0, numBytes); } return output.toByteArray(); }
@Override public KeyedStateRow map(Tuple2<K, V> t) throws Exception { int keyGroup = KeyGroupRangeAssignment.assignToKeyGroup(t.f0, maxParallelism); ByteArrayOutputStreamWithPos os = new ByteArrayOutputStreamWithPos(); DataOutputViewStreamWrapper ov = new DataOutputViewStreamWrapper(os); RocksDBUtils.writeKeyGroup(keyGroup, keygroupPrefixBytes, ov); RocksDBUtils.writeKey(t.f0, keySerializer, os, ov, false); RocksDBUtils.writeNameSpace(VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, os, ov, false); os.close(); return new KeyedStateRow(stateName, os.toByteArray(), InstantiationUtil.serializeToByteArray(valueSerializer, t.f1)); } }
@Before public void setup() { stream = new ByteArrayOutputStreamWithPos(BUFFER_SIZE); }
/** * Test setting negative position. */ @Test public void testSetNegativePosition() throws Exception { stream.write(new byte[BUFFER_SIZE]); thrown.expect(IllegalArgumentException.class); thrown.expectMessage("Position out of bounds"); stream.setPosition(-1); }
@Override public void flush() throws IOException { os.flush(); }
/** * Decode a String value. */ public String decodeString(DataInputView source, Order ord) throws IOException { if (reuseBuffer == null) { reuseBuffer = new byte[BUFFER_SIZE]; } if (reuseBaos == null) { reuseBaos = new ByteArrayOutputStreamWithPos(); } reuseBaos.reset(); int len; while ((len = source.read(reuseBuffer)) != -1) { reuseBaos.write(reuseBuffer, 0, len); } if (ord == Order.DESCENDING) { // DESCENDING ordered string requires a termination bit to preserve // sort-order semantics of empty values. reuseBaos.setPosition(reuseBaos.size() - 1); } return new String(ord.apply(reuseBaos.toByteArray()), UTF8); }