final byte[] serializedRecoverable = serializer.serialize(recoverables.get(persistName)); final SimpleVersionedSerializer<RecoverableWriter.ResumeRecoverable> deserializer = newWriter.getResumeRecoverableSerializer(); final RecoverableWriter.ResumeRecoverable recoveredRecoverable = deserializer.deserialize(serializer.getVersion(), serializedRecoverable);
checkNotNull(datum, "datum"); final byte[] data = serializer.serialize(datum); final byte[] versionAndData = new byte[data.length + 8]; final int version = serializer.getVersion(); versionAndData[0] = (byte) (version >> 24); versionAndData[1] = (byte) (version >> 16);
return serializer.deserialize(version, dataOnly);
/** * Serializes the version and datum into a stream. * * <p>Data serialized via this method can be deserialized via * {@link #readVersionAndDeSerialize(SimpleVersionedSerializer, DataInputView)}. * * <p>The first four bytes will be occupied by the version, as returned by * {@link SimpleVersionedSerializer#getVersion()}. The remaining bytes will be the serialized * datum, as produced by {@link SimpleVersionedSerializer#serialize(Object)}, plus its length. * The resulting array will hence be eight bytes larger than the serialized datum. * * @param serializer The serializer to serialize the datum with. * @param datum The datum to serialize. * @param out The stream to serialize to. */ public static <T> void writeVersionAndSerialize(SimpleVersionedSerializer<T> serializer, T datum, DataOutputView out) throws IOException { checkNotNull(serializer, "serializer"); checkNotNull(datum, "datum"); checkNotNull(out, "out"); final byte[] data = serializer.serialize(datum); out.writeInt(serializer.getVersion()); out.writeInt(data.length); out.write(data); }
/** * Deserializes the version and datum from a stream. * * <p>This method deserializes data serialized via * {@link #writeVersionAndSerialize(SimpleVersionedSerializer, Object, DataOutputView)}. * * <p>The first four bytes will be interpreted as the version. The next four bytes will be * interpreted as the length of the datum bytes, then length-many bytes will be read. * Finally, the datum is deserialized via the {@link SimpleVersionedSerializer#deserialize(int, byte[])} * method. * * @param serializer The serializer to serialize the datum with. * @param in The stream to deserialize from. */ public static <T> T readVersionAndDeSerialize(SimpleVersionedSerializer<T> serializer, DataInputView in) throws IOException { checkNotNull(serializer, "serializer"); checkNotNull(in, "in"); final int version = in.readInt(); final int length = in.readInt(); final byte[] data = new byte[length]; in.readFully(data); return serializer.deserialize(version, data); }
@VisibleForTesting void serializeV1(BucketState<BucketID> state, DataOutputView out) throws IOException { SimpleVersionedSerialization.writeVersionAndSerialize(bucketIdSerializer, state.getBucketId(), out); out.writeUTF(state.getBucketPath().toString()); out.writeLong(state.getInProgressFileCreationTime()); // put the current open part file if (state.hasInProgressResumableFile()) { final RecoverableWriter.ResumeRecoverable resumable = state.getInProgressResumableFile(); out.writeBoolean(true); SimpleVersionedSerialization.writeVersionAndSerialize(resumableSerializer, resumable, out); } else { out.writeBoolean(false); } // put the map of pending files per checkpoint final Map<Long, List<RecoverableWriter.CommitRecoverable>> pendingCommitters = state.getCommittableFilesPerCheckpoint(); // manually keep the version here to safe some bytes out.writeInt(commitableSerializer.getVersion()); out.writeInt(pendingCommitters.size()); for (Entry<Long, List<RecoverableWriter.CommitRecoverable>> resumablesForCheckpoint : pendingCommitters.entrySet()) { List<RecoverableWriter.CommitRecoverable> resumables = resumablesForCheckpoint.getValue(); out.writeLong(resumablesForCheckpoint.getKey()); out.writeInt(resumables.size()); for (RecoverableWriter.CommitRecoverable resumable : resumables) { byte[] serialized = commitableSerializer.serialize(resumable); out.writeInt(serialized.length); out.write(serialized); } } }
@Test public void testCommitAfterRecovery() throws Exception { final Path testDir = getBasePathForTest(); final Path path = new Path(testDir, "part-0"); final RecoverableWriter initWriter = getNewFileSystemWriter(); final RecoverableWriter.CommitRecoverable recoverable; try (final RecoverableFsDataOutputStream stream = initWriter.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); stream.persist(); stream.persist(); // and write some more data stream.write(testData2.getBytes(StandardCharsets.UTF_8)); recoverable = stream.closeForCommit().getRecoverable(); } final byte[] serializedRecoverable = initWriter.getCommitRecoverableSerializer().serialize(recoverable); // get a new serializer from a new writer to make sure that no pre-initialized state leaks in. final RecoverableWriter newWriter = getNewFileSystemWriter(); final SimpleVersionedSerializer<RecoverableWriter.CommitRecoverable> deserializer = newWriter.getCommitRecoverableSerializer(); final RecoverableWriter.CommitRecoverable recoveredRecoverable = deserializer.deserialize(deserializer.getVersion(), serializedRecoverable); final RecoverableFsDataOutputStream.Committer committer = newWriter.recoverForCommit(recoveredRecoverable); committer.commitAfterRecovery(); Map<Path, String> files = getFileContentByPath(testDir); Assert.assertEquals(1L, files.size()); for (Map.Entry<Path, String> fileContents : files.entrySet()) { Assert.assertEquals("part-0", fileContents.getKey().getName()); Assert.assertEquals(testData1 + testData2, fileContents.getValue()); } }
final byte[] bytes = new byte[in.readInt()]; in.readFully(bytes); resumables.add(commitableSerializer.deserialize(committableVersion, bytes));
checkNotNull(datum, "datum"); final byte[] data = serializer.serialize(datum); final byte[] versionAndData = new byte[data.length + 8]; final int version = serializer.getVersion(); versionAndData[0] = (byte) (version >> 24); versionAndData[1] = (byte) (version >> 16);
return serializer.deserialize(version, dataOnly);
/** * Serializes the version and datum into a stream. * * <p>Data serialized via this method can be deserialized via * {@link #readVersionAndDeSerialize(SimpleVersionedSerializer, DataInputView)}. * * <p>The first four bytes will be occupied by the version, as returned by * {@link SimpleVersionedSerializer#getVersion()}. The remaining bytes will be the serialized * datum, as produced by {@link SimpleVersionedSerializer#serialize(Object)}, plus its length. * The resulting array will hence be eight bytes larger than the serialized datum. * * @param serializer The serializer to serialize the datum with. * @param datum The datum to serialize. * @param out The stream to serialize to. */ public static <T> void writeVersionAndSerialize(SimpleVersionedSerializer<T> serializer, T datum, DataOutputView out) throws IOException { checkNotNull(serializer, "serializer"); checkNotNull(datum, "datum"); checkNotNull(out, "out"); final byte[] data = serializer.serialize(datum); out.writeInt(serializer.getVersion()); out.writeInt(data.length); out.write(data); }
/** * Deserializes the version and datum from a stream. * * <p>This method deserializes data serialized via * {@link #writeVersionAndSerialize(SimpleVersionedSerializer, Object, DataOutputView)}. * * <p>The first four bytes will be interpreted as the version. The next four bytes will be * interpreted as the length of the datum bytes, then length-many bytes will be read. * Finally, the datum is deserialized via the {@link SimpleVersionedSerializer#deserialize(int, byte[])} * method. * * @param serializer The serializer to serialize the datum with. * @param in The stream to deserialize from. */ public static <T> T readVersionAndDeSerialize(SimpleVersionedSerializer<T> serializer, DataInputView in) throws IOException { checkNotNull(serializer, "serializer"); checkNotNull(in, "in"); final int version = in.readInt(); final int length = in.readInt(); final byte[] data = new byte[length]; in.readFully(data); return serializer.deserialize(version, data); }
@VisibleForTesting void serializeV1(BucketState<BucketID> state, DataOutputView out) throws IOException { SimpleVersionedSerialization.writeVersionAndSerialize(bucketIdSerializer, state.getBucketId(), out); out.writeUTF(state.getBucketPath().toString()); out.writeLong(state.getInProgressFileCreationTime()); // put the current open part file if (state.hasInProgressResumableFile()) { final RecoverableWriter.ResumeRecoverable resumable = state.getInProgressResumableFile(); out.writeBoolean(true); SimpleVersionedSerialization.writeVersionAndSerialize(resumableSerializer, resumable, out); } else { out.writeBoolean(false); } // put the map of pending files per checkpoint final Map<Long, List<RecoverableWriter.CommitRecoverable>> pendingCommitters = state.getCommittableFilesPerCheckpoint(); // manually keep the version here to safe some bytes out.writeInt(commitableSerializer.getVersion()); out.writeInt(pendingCommitters.size()); for (Entry<Long, List<RecoverableWriter.CommitRecoverable>> resumablesForCheckpoint : pendingCommitters.entrySet()) { List<RecoverableWriter.CommitRecoverable> resumables = resumablesForCheckpoint.getValue(); out.writeLong(resumablesForCheckpoint.getKey()); out.writeInt(resumables.size()); for (RecoverableWriter.CommitRecoverable resumable : resumables) { byte[] serialized = commitableSerializer.serialize(resumable); out.writeInt(serialized.length); out.write(serialized); } } }
private static <T> T deserializeState(MasterState state, MasterTriggerRestoreHook<?> hook) throws FlinkException { @SuppressWarnings("unchecked") final MasterTriggerRestoreHook<T> typedHook = (MasterTriggerRestoreHook<T>) hook; final String id = hook.getIdentifier(); try { final SimpleVersionedSerializer<T> deserializer = typedHook.createCheckpointDataSerializer(); if (deserializer == null) { throw new FlinkException("null serializer for state of hook " + hook.getIdentifier()); } return deserializer.deserialize(state.version(), state.bytes()); } catch (Throwable t) { throw new FlinkException("Cannot deserialize state for master hook '" + id + '\'', t); } }
@VisibleForTesting void serializeV1(BucketState<BucketID> state, DataOutputView out) throws IOException { SimpleVersionedSerialization.writeVersionAndSerialize(bucketIdSerializer, state.getBucketId(), out); out.writeUTF(state.getBucketPath().toString()); out.writeLong(state.getInProgressFileCreationTime()); // put the current open part file if (state.hasInProgressResumableFile()) { final RecoverableWriter.ResumeRecoverable resumable = state.getInProgressResumableFile(); out.writeBoolean(true); SimpleVersionedSerialization.writeVersionAndSerialize(resumableSerializer, resumable, out); } else { out.writeBoolean(false); } // put the map of pending files per checkpoint final Map<Long, List<RecoverableWriter.CommitRecoverable>> pendingCommitters = state.getCommittableFilesPerCheckpoint(); // manually keep the version here to safe some bytes out.writeInt(commitableSerializer.getVersion()); out.writeInt(pendingCommitters.size()); for (Entry<Long, List<RecoverableWriter.CommitRecoverable>> resumablesForCheckpoint : pendingCommitters.entrySet()) { List<RecoverableWriter.CommitRecoverable> resumables = resumablesForCheckpoint.getValue(); out.writeLong(resumablesForCheckpoint.getKey()); out.writeInt(resumables.size()); for (RecoverableWriter.CommitRecoverable resumable : resumables) { byte[] serialized = commitableSerializer.serialize(resumable); out.writeInt(serialized.length); out.write(serialized); } } }
private static <T> T deserializeState(MasterState state, MasterTriggerRestoreHook<?> hook) throws FlinkException { @SuppressWarnings("unchecked") final MasterTriggerRestoreHook<T> typedHook = (MasterTriggerRestoreHook<T>) hook; final String id = hook.getIdentifier(); try { final SimpleVersionedSerializer<T> deserializer = typedHook.createCheckpointDataSerializer(); if (deserializer == null) { throw new FlinkException("null serializer for state of hook " + hook.getIdentifier()); } return deserializer.deserialize(state.version(), state.bytes()); } catch (Throwable t) { throw new FlinkException("Cannot deserialize state for master hook '" + id + '\'', t); } }
final int version = serializer.getVersion(); final byte[] bytes = serializer.serialize(result);
private static <T> T deserializeState(MasterState state, MasterTriggerRestoreHook<?> hook) throws FlinkException { @SuppressWarnings("unchecked") final MasterTriggerRestoreHook<T> typedHook = (MasterTriggerRestoreHook<T>) hook; final String id = hook.getIdentifier(); try { final SimpleVersionedSerializer<T> deserializer = typedHook.createCheckpointDataSerializer(); if (deserializer == null) { throw new FlinkException("null serializer for state of hook " + hook.getIdentifier()); } return deserializer.deserialize(state.version(), state.bytes()); } catch (Throwable t) { throw new FlinkException("Cannot deserialize state for master hook '" + id + '\'', t); } }
final int version = serializer.getVersion(); final byte[] bytes = serializer.serialize(result);
private static <T> T deserializeState(MasterState state, MasterTriggerRestoreHook<?> hook) throws FlinkException { @SuppressWarnings("unchecked") final MasterTriggerRestoreHook<T> typedHook = (MasterTriggerRestoreHook<T>) hook; final String id = hook.getIdentifier(); try { final SimpleVersionedSerializer<T> deserializer = typedHook.createCheckpointDataSerializer(); if (deserializer == null) { throw new FlinkException("null serializer for state of hook " + hook.getIdentifier()); } return deserializer.deserialize(state.version(), state.bytes()); } catch (Throwable t) { throw new FlinkException("Cannot deserialize state for master hook '" + id + '\'', t); } }