final byte[] versionAndData = new byte[data.length + 8]; final int version = serializer.getVersion(); versionAndData[0] = (byte) (version >> 24); versionAndData[1] = (byte) (version >> 16);
@VisibleForTesting void serializeV1(BucketState<BucketID> state, DataOutputView out) throws IOException { SimpleVersionedSerialization.writeVersionAndSerialize(bucketIdSerializer, state.getBucketId(), out); out.writeUTF(state.getBucketPath().toString()); out.writeLong(state.getInProgressFileCreationTime()); // put the current open part file if (state.hasInProgressResumableFile()) { final RecoverableWriter.ResumeRecoverable resumable = state.getInProgressResumableFile(); out.writeBoolean(true); SimpleVersionedSerialization.writeVersionAndSerialize(resumableSerializer, resumable, out); } else { out.writeBoolean(false); } // put the map of pending files per checkpoint final Map<Long, List<RecoverableWriter.CommitRecoverable>> pendingCommitters = state.getCommittableFilesPerCheckpoint(); // manually keep the version here to safe some bytes out.writeInt(commitableSerializer.getVersion()); out.writeInt(pendingCommitters.size()); for (Entry<Long, List<RecoverableWriter.CommitRecoverable>> resumablesForCheckpoint : pendingCommitters.entrySet()) { List<RecoverableWriter.CommitRecoverable> resumables = resumablesForCheckpoint.getValue(); out.writeLong(resumablesForCheckpoint.getKey()); out.writeInt(resumables.size()); for (RecoverableWriter.CommitRecoverable resumable : resumables) { byte[] serialized = commitableSerializer.serialize(resumable); out.writeInt(serialized.length); out.write(serialized); } } }
/** * Serializes the version and datum into a stream. * * <p>Data serialized via this method can be deserialized via * {@link #readVersionAndDeSerialize(SimpleVersionedSerializer, DataInputView)}. * * <p>The first four bytes will be occupied by the version, as returned by * {@link SimpleVersionedSerializer#getVersion()}. The remaining bytes will be the serialized * datum, as produced by {@link SimpleVersionedSerializer#serialize(Object)}, plus its length. * The resulting array will hence be eight bytes larger than the serialized datum. * * @param serializer The serializer to serialize the datum with. * @param datum The datum to serialize. * @param out The stream to serialize to. */ public static <T> void writeVersionAndSerialize(SimpleVersionedSerializer<T> serializer, T datum, DataOutputView out) throws IOException { checkNotNull(serializer, "serializer"); checkNotNull(datum, "datum"); checkNotNull(out, "out"); final byte[] data = serializer.serialize(datum); out.writeInt(serializer.getVersion()); out.writeInt(data.length); out.write(data); }
final SimpleVersionedSerializer<RecoverableWriter.ResumeRecoverable> deserializer = newWriter.getResumeRecoverableSerializer(); final RecoverableWriter.ResumeRecoverable recoveredRecoverable = deserializer.deserialize(serializer.getVersion(), serializedRecoverable);
@Test public void testCommitAfterRecovery() throws Exception { final Path testDir = getBasePathForTest(); final Path path = new Path(testDir, "part-0"); final RecoverableWriter initWriter = getNewFileSystemWriter(); final RecoverableWriter.CommitRecoverable recoverable; try (final RecoverableFsDataOutputStream stream = initWriter.open(path)) { stream.write(testData1.getBytes(StandardCharsets.UTF_8)); stream.persist(); stream.persist(); // and write some more data stream.write(testData2.getBytes(StandardCharsets.UTF_8)); recoverable = stream.closeForCommit().getRecoverable(); } final byte[] serializedRecoverable = initWriter.getCommitRecoverableSerializer().serialize(recoverable); // get a new serializer from a new writer to make sure that no pre-initialized state leaks in. final RecoverableWriter newWriter = getNewFileSystemWriter(); final SimpleVersionedSerializer<RecoverableWriter.CommitRecoverable> deserializer = newWriter.getCommitRecoverableSerializer(); final RecoverableWriter.CommitRecoverable recoveredRecoverable = deserializer.deserialize(deserializer.getVersion(), serializedRecoverable); final RecoverableFsDataOutputStream.Committer committer = newWriter.recoverForCommit(recoveredRecoverable); committer.commitAfterRecovery(); Map<Path, String> files = getFileContentByPath(testDir); Assert.assertEquals(1L, files.size()); for (Map.Entry<Path, String> fileContents : files.entrySet()) { Assert.assertEquals("part-0", fileContents.getKey().getName()); Assert.assertEquals(testData1 + testData2, fileContents.getValue()); } }
final byte[] versionAndData = new byte[data.length + 8]; final int version = serializer.getVersion(); versionAndData[0] = (byte) (version >> 24); versionAndData[1] = (byte) (version >> 16);
/** * Serializes the version and datum into a stream. * * <p>Data serialized via this method can be deserialized via * {@link #readVersionAndDeSerialize(SimpleVersionedSerializer, DataInputView)}. * * <p>The first four bytes will be occupied by the version, as returned by * {@link SimpleVersionedSerializer#getVersion()}. The remaining bytes will be the serialized * datum, as produced by {@link SimpleVersionedSerializer#serialize(Object)}, plus its length. * The resulting array will hence be eight bytes larger than the serialized datum. * * @param serializer The serializer to serialize the datum with. * @param datum The datum to serialize. * @param out The stream to serialize to. */ public static <T> void writeVersionAndSerialize(SimpleVersionedSerializer<T> serializer, T datum, DataOutputView out) throws IOException { checkNotNull(serializer, "serializer"); checkNotNull(datum, "datum"); checkNotNull(out, "out"); final byte[] data = serializer.serialize(datum); out.writeInt(serializer.getVersion()); out.writeInt(data.length); out.write(data); }
@VisibleForTesting void serializeV1(BucketState<BucketID> state, DataOutputView out) throws IOException { SimpleVersionedSerialization.writeVersionAndSerialize(bucketIdSerializer, state.getBucketId(), out); out.writeUTF(state.getBucketPath().toString()); out.writeLong(state.getInProgressFileCreationTime()); // put the current open part file if (state.hasInProgressResumableFile()) { final RecoverableWriter.ResumeRecoverable resumable = state.getInProgressResumableFile(); out.writeBoolean(true); SimpleVersionedSerialization.writeVersionAndSerialize(resumableSerializer, resumable, out); } else { out.writeBoolean(false); } // put the map of pending files per checkpoint final Map<Long, List<RecoverableWriter.CommitRecoverable>> pendingCommitters = state.getCommittableFilesPerCheckpoint(); // manually keep the version here to safe some bytes out.writeInt(commitableSerializer.getVersion()); out.writeInt(pendingCommitters.size()); for (Entry<Long, List<RecoverableWriter.CommitRecoverable>> resumablesForCheckpoint : pendingCommitters.entrySet()) { List<RecoverableWriter.CommitRecoverable> resumables = resumablesForCheckpoint.getValue(); out.writeLong(resumablesForCheckpoint.getKey()); out.writeInt(resumables.size()); for (RecoverableWriter.CommitRecoverable resumable : resumables) { byte[] serialized = commitableSerializer.serialize(resumable); out.writeInt(serialized.length); out.write(serialized); } } }
@VisibleForTesting void serializeV1(BucketState<BucketID> state, DataOutputView out) throws IOException { SimpleVersionedSerialization.writeVersionAndSerialize(bucketIdSerializer, state.getBucketId(), out); out.writeUTF(state.getBucketPath().toString()); out.writeLong(state.getInProgressFileCreationTime()); // put the current open part file if (state.hasInProgressResumableFile()) { final RecoverableWriter.ResumeRecoverable resumable = state.getInProgressResumableFile(); out.writeBoolean(true); SimpleVersionedSerialization.writeVersionAndSerialize(resumableSerializer, resumable, out); } else { out.writeBoolean(false); } // put the map of pending files per checkpoint final Map<Long, List<RecoverableWriter.CommitRecoverable>> pendingCommitters = state.getCommittableFilesPerCheckpoint(); // manually keep the version here to safe some bytes out.writeInt(commitableSerializer.getVersion()); out.writeInt(pendingCommitters.size()); for (Entry<Long, List<RecoverableWriter.CommitRecoverable>> resumablesForCheckpoint : pendingCommitters.entrySet()) { List<RecoverableWriter.CommitRecoverable> resumables = resumablesForCheckpoint.getValue(); out.writeLong(resumablesForCheckpoint.getKey()); out.writeInt(resumables.size()); for (RecoverableWriter.CommitRecoverable resumable : resumables) { byte[] serialized = commitableSerializer.serialize(resumable); out.writeInt(serialized.length); out.write(serialized); } } }
final int version = serializer.getVersion(); final byte[] bytes = serializer.serialize(result);
final int version = serializer.getVersion(); final byte[] bytes = serializer.serialize(result);
final int version = serializer.getVersion(); final byte[] bytes = serializer.serialize(result);
final int version = serializer.getVersion(); final byte[] bytes = serializer.serialize(result);