private void validateHeader(int keyLength, int valueLength) throws SerializationException { if (keyLength <= 0 || keyLength > MAX_KEY_LENGTH || (valueLength < 0 && valueLength != NO_VALUE) || keyLength + valueLength > MAX_SERIALIZATION_LENGTH) { throw new SerializationException(String.format("Read header with invalid data. KeyLength=%s, ValueLength=%s", keyLength, valueLength)); } }
/** * Verifies that the given condition is true. If not, throws a SerializationException. */ void ensureCondition(boolean condition, String messageFormat, Object... args) throws SerializationException { if (!condition) { throw new SerializationException(String.format(messageFormat, args)); } }
@Override public void close() throws IOException { // Skip over the remaining bytes. Do not close the underlying InputStream. if (this.remaining > 0) { int toSkip = this.remaining; long skipped = skip(toSkip); if (skipped != toSkip) { throw new SerializationException(String.format("Read %d fewer byte(s) than expected only able to skip %d.", toSkip, skipped)); } } else if (this.remaining < 0) { throw new SerializationException(String.format("Read more bytes than expected (%d).", -this.remaining)); } }
private void startNewRecordInCurrentFrame(boolean firstRecordEntry) throws SerializationException { if (!this.currentFrame.startNewEntry(firstRecordEntry)) { throw new SerializationException("Unable to start a new record."); } this.hasDataInCurrentFrame = true; }
@Override public void processResultComplete() { if (!this.result.isDone()) { // We've reached the end of the read but couldn't find anything. processError(new SerializationException("Reached the end of the ReadResult but unable to read desired data.")); } }
private void read00(RevisionDataInput input, ContainerMetadataUpdateTransaction t) throws IOException { int containerId = input.readCompactInt(); if (t.containerId != containerId) { throw new SerializationException(String.format("Invalid ContainerId. Expected '%d', actual '%d'.", t.containerId, containerId)); } input.readCollection(s -> readSegmentMetadata00(s, t)); }
@Override public void close() throws IOException { // We do not want to close the underlying Stream as it may be reused. if (this.length != size()) { // Check if we wrote the number of bytes we declared, otherwise we will have problems upon deserializing. throw new SerializationException(String.format("Unexpected number of bytes written. Declared: %d, written: %d.", this.length, size())); } else if (requiresExplicitLength()) { // We haven't written anything nor declared a length. Write the length prior to exiting. length(0); } }
private DataFrame.DataFrameEntryIterator getNextFrame() throws DurableDataLogException, IOException { DurableDataLog.ReadItem nextItem = this.reader.getNext(); if (nextItem == null) { // We have reached the end. Stop here. return null; } DataFrame.DataFrameEntryIterator frameContents; try { frameContents = DataFrame.read(nextItem.getPayload(), nextItem.getLength(), nextItem.getAddress()); } catch (SerializationException ex) { throw new SerializationException(String.format("Unable to deserialize DataFrame. LastReadFrameSequence = %d.", this.lastReadFrameSequence), ex); } long sequence = nextItem.getAddress().getSequence(); if (sequence <= this.lastReadFrameSequence) { // FrameSequence must be a strictly monotonically increasing number. throw new SerializationException(String.format("Found DataFrame out of order. Expected frame sequence greater than %d, found %d.", this.lastReadFrameSequence, sequence)); } this.lastReadFrameSequence = sequence; return frameContents; }
/** * Interprets the given InputStream as a DataFrame and returns a DataFrameEntryIterator for the entries serialized * in it. * * @param source The InputStream to read from. * @param length The size of the inputStream. * @param address The DataFrame's address. * @return A new DataFrameEntryIterator. * @throws IOException If unable to parse the DataFrame's header from the InputStream. */ public static DataFrameEntryIterator read(InputStream source, int length, LogAddress address) throws IOException { // Check to see that we have enough bytes in the InputStream. ReadFrameHeader header = new ReadFrameHeader(source); if (length < ReadFrameHeader.SERIALIZATION_LENGTH + header.getContentLength()) { throw new SerializationException(String.format("Given buffer has insufficient number of bytes for this DataFrame. Expected %d, actual %d.", ReadFrameHeader.SERIALIZATION_LENGTH + header.getContentLength(), length)); } BoundedInputStream contents = new BoundedInputStream(source, header.getContentLength()); return new DataFrameEntryIterator(contents, address, ReadFrameHeader.SERIALIZATION_LENGTH); }
private DataFrame.DataFrameEntry getNextFrameEntry() throws DurableDataLogException, IOException { // Check to see if we are in the middle of a frame, in which case, just return the next element. DataFrame.DataFrameEntry result; if (this.currentFrameContents != null) { result = this.currentFrameContents.getNext(); if (result != null) { return result; } } // We need to fetch a new frame. this.currentFrameContents = getNextFrame(); if (this.currentFrameContents == null) { // No more frames to retrieve return null; } else { // We just got a new frame. log.debug("{}: Read DataFrame (Address = {}, Length = {}).", this.traceObjectId, this.currentFrameContents.getFrameAddress(), this.currentFrameContents.getLength()); result = this.currentFrameContents.getNext(); if (result != null) { return result; } else { // The DataFrameEnumerator should not return empty frames. We can either go in a loop and try to get next, // or throw (which is correct, since we rely on DataFrameEnumerator to behave correctly). throw new SerializationException("Found empty DataFrame when non-empty was expected."); } } }
@Override public void write(int b) throws IOException { Exceptions.checkNotClosed(this.closed, this); Preconditions.checkState(this.currentFrame != null, "No current frame exists. Most likely no record is started."); int attemptCount = 0; int totalBytesWritten = 0; while (totalBytesWritten == 0 && attemptCount < 2) { // We attempt to write 1 byte. If append() says it wrote 0 bytes, it means the current frame is full. Seal it and create a new one. totalBytesWritten += this.currentFrame.append((byte) b); if (totalBytesWritten == 0) { this.currentFrame.endEntry(false); // Close the current entry, and indicate it is not the last one of the record. flush(); createNewFrame(); startNewRecordInCurrentFrame(false); } attemptCount++; } if (totalBytesWritten == 0) { throw new SerializationException("Unable to make progress in serializing to DataFrame."); } }
@Override public DataFrameEntry getNext() throws IOException { closeLast(); if (reachedEnd()) { return null; } // Integrity check. This means that we have a corrupt frame. if (this.contents.getRemaining() < EntryHeader.HEADER_SIZE) { throw new SerializationException(String.format("Data Frame is corrupt. InputStream has insufficient bytes for a new Entry Header (%d).", this.contents.getRemaining())); } // Determine the length of the next record and advance the position by the appropriate amount of bytes. ReadEntryHeader header = new ReadEntryHeader(this.contents); // Integrity check. This means that we have a corrupt frame. if (this.contents.getRemaining() < header.getEntryLength()) { throw new SerializationException(String.format("Data Frame is corrupt. Found Entry Length %d which cannot fit in the Frame's remaining length of %d.", header.getEntryLength(), this.contents.getRemaining())); } // Get the result contents && advance the positions. int frameOffset = this.bufferOffset + this.contents.getBound() - this.contents.getRemaining(); BoundedInputStream resultContents = this.contents.subStream(header.getEntryLength()); this.lastEntryContents = resultContents; return new DataFrameEntry(header, resultContents, this.frameAddress, reachedEnd(), frameOffset); }
throw new SerializationException(String.format("Found a DataFrameRecord which is not marked as " + "'First Record Entry', but no active record is being read. DataFrameAddress = %s", nextEntry.getFrameAddress()));
/** * Tests the case when the DataFrameReader reads from a log and it encounters LogItem SerializationExceptions. */ @Test public void testReadsWithDeserializationFailure() throws Exception { int failDeserializationEvery = 11; // Fail deserialization every X records (write-wise). ArrayList<TestLogItem> records = DataFrameTestHelpers.generateLogItems(100, SMALL_RECORD_MIN_SIZE, SMALL_RECORD_MAX_SIZE, 0); records.addAll(DataFrameTestHelpers.generateLogItems(100, LARGE_RECORD_MIN_SIZE, LARGE_RECORD_MAX_SIZE, records.size())); try (TestDurableDataLog dataLog = TestDurableDataLog.create(CONTAINER_ID, FRAME_SIZE, executorService())) { dataLog.initialize(TIMEOUT); BiConsumer<Throwable, DataFrameBuilder.CommitArgs> errorCallback = (ex, a) -> Assert.fail(String.format("Unexpected error occurred upon commit. %s", ex)); val args = new DataFrameBuilder.Args(Callbacks::doNothing, Callbacks::doNothing, errorCallback, executorService()); try (DataFrameBuilder<TestLogItem> b = new DataFrameBuilder<>(dataLog, SERIALIZER, args)) { for (TestLogItem r : records) { b.append(r); } } ErrorInjector<SerializationException> errorInjector = new ErrorInjector<>( count -> count % failDeserializationEvery == 0, () -> new SerializationException("TestLogItem.deserialize intentional")); TestSerializer logItemFactory = new TestSerializer(); logItemFactory.setDeserializationErrorInjector(errorInjector); testReadWithException(dataLog, logItemFactory, ex -> ex instanceof DataCorruptionException); } }
private void setCurrentFrameEntry(DataFrame.DataFrameEntry nextEntry) throws IOException { long dataFrameSequence = nextEntry.getFrameAddress().getSequence(); LogAddress lastUsedAddress = this.currentRecordBuilder.getLastUsedDataFrameAddress(); if (lastUsedAddress != null && dataFrameSequence < lastUsedAddress.getSequence()) { throw new SerializationException(String.format("Invalid DataFrameSequence. Expected at least '%d', found '%d'.", lastUsedAddress.getSequence(), dataFrameSequence)); } if (nextEntry.isLastEntryInDataFrame()) { // This is the last segment in this DataFrame, so we need to set the lastFullDataFrameAddress to the right value. this.currentRecordBuilder.lastFullDataFrameAddress(nextEntry.getFrameAddress()); } this.currentRecordBuilder.lastUsedDataFrameAddress(nextEntry.getFrameAddress()); this.currentRecordBuilder.lastFrameEntry(nextEntry.isLastEntryInDataFrame()); this.currentEntry = nextEntry; if (this.currentEntry.isLastRecordEntry()) { this.hasReadAnyData = true; } this.currentRecordBuilder.withEntry(nextEntry.getFrameAddress(), nextEntry.getFrameOffset(), nextEntry.getLength(), nextEntry.isLastEntryInDataFrame()); }