private void ensureOffset(RollingSegmentHandle handle, long offset) throws StreamSegmentException { if (offset != handle.length()) { // Force-refresh the handle to make sure it is still in sync with reality. Make sure we open a read handle // so that we don't force any sort of fencing during this process. val refreshedHandle = openHandle(handle.getSegmentName(), true); handle.refresh(refreshedHandle); log.debug("Handle refreshed: {}.", handle); if (offset != handle.length()) { // Still in disagreement; throw exception. throw new BadOffsetException(handle.getSegmentName(), handle.length(), offset); } } }
private void serializeBeginConcat(RollingSegmentHandle targetHandle, RollingSegmentHandle sourceHandle) throws StreamSegmentException { byte[] updateData = HandleSerializer.serializeConcat(sourceHandle.chunks().size(), targetHandle.length()); updateHandle(targetHandle, updateData); }
private void checkConcatResult(RollingStorage s, RollingSegmentHandle targetHandle, String sourceSegmentName, int expectedChunkCount, int expectedLength) throws Exception { Assert.assertFalse("Expecting the source segment to not exist anymore.", s.exists(sourceSegmentName)); Assert.assertEquals("Unexpected number of SegmentChunks in target.", expectedChunkCount, targetHandle.chunks().size()); Assert.assertEquals("Unexpected target length.", expectedLength, targetHandle.length()); // Reload the handle and verify nothing strange happened in Storage. val targetHandle2 = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME); Assert.assertEquals("Unexpected number of SegmentChunks in reloaded target handle.", expectedChunkCount, targetHandle2.chunks().size()); Assert.assertEquals("Unexpected reloaded target length.", targetHandle.length(), targetHandle2.length()); }
private void createChunk(RollingSegmentHandle handle) throws StreamSegmentException { // Create new active SegmentChunk, only after which serialize the handle update and update the handle. // We ignore if the SegmentChunk exists and is empty - that's most likely due to a previous failed attempt. long segmentLength = handle.length(); SegmentChunk newSegmentChunk = SegmentChunk.forSegment(handle.getSegmentName(), segmentLength); try { this.baseStorage.create(newSegmentChunk.getName()); } catch (StreamSegmentExistsException ex) { checkIfEmptyAndNotSealed(ex, newSegmentChunk.getName()); } serializeNewChunk(handle, newSegmentChunk); val activeHandle = this.baseStorage.openWrite(newSegmentChunk.getName()); handle.addChunk(newSegmentChunk, activeHandle); log.debug("Created new SegmentChunk '{}' for '{}'.", newSegmentChunk, handle); }
@Override public SegmentProperties getStreamSegmentInfo(String segmentName) throws StreamSegmentException { val handle = (RollingSegmentHandle) openRead(segmentName); return StreamSegmentInformation .builder() .name(handle.getSegmentName()) .sealed(handle.isSealed()) .length(handle.length()) .build(); }
@Override public void truncate(SegmentHandle handle, long truncationOffset) throws StreamSegmentException { // Delete all SegmentChunks which are entirely before the truncation offset. RollingSegmentHandle h = asReadableHandle(handle); ensureNotDeleted(h); // The only acceptable case where we allow a read-only handle is if the Segment is sealed, since openWrite() will // only return a read-only handle in that case. Preconditions.checkArgument(h.isSealed() || !h.isReadOnly(), "Can only truncate with a read-only handle if the Segment is Sealed."); if (h.getHeaderHandle() == null) { // No header means the Segment is made up of a single SegmentChunk. We can't do anything. return; } long traceId = LoggerHelpers.traceEnter(log, "truncate", h, truncationOffset); Preconditions.checkArgument(truncationOffset >= 0 && truncationOffset <= h.length(), "truncationOffset must be non-negative and at most the length of the Segment."); val last = h.lastChunk(); if (last != null && canTruncate(last, truncationOffset) && !h.isSealed()) { // If we were asked to truncate the entire (non-sealed) Segment, then rollover at this point so we can delete // all existing data. rollover(h); // We are free to delete all chunks. deleteChunks(h, s -> canTruncate(s, truncationOffset)); } else { // Either we were asked not to truncate the whole segment, or we were, and the Segment is sealed. If the latter, // then the Header is also sealed, we could not have done a quick rollover; as such we have no option but to // preserve the last chunk so that we can recalculate the length of the Segment if we need it again. deleteChunks(h, s -> canTruncate(s, truncationOffset) && s.getLastOffset() < h.length()); } LoggerHelpers.traceLeave(log, "truncate", traceId, h, truncationOffset); }
os.write(serialize(source)); for (int i = 0; i < concatCount; i++) { os.write(HandleSerializer.serializeConcat(chunkCount, source.length())); if (i % failConcatEvery != 0) { os.write(serialize(concatHandle)); source.addChunks(concatHandle.chunks().stream() .map(s -> s.withNewOffset(s.getStartOffset() + source.length())).collect(Collectors.toList())); val chunk = new SegmentChunk(StreamSegmentNameUtils.getSegmentChunkName(source.getSegmentName(), source.length()), source.length()); chunk.setLength(i + 1); source.addChunks(Collections.singletonList(chunk));
ex -> ex instanceof IllegalArgumentException); Assert.assertEquals("Not expecting any SegmentChunks to be added.", 0, h.chunks().size()); Assert.assertEquals("Unexpected length().", 0, h.length()); h.addChunks(validList); AssertExtensions.assertListEquals("Unexpected list of SegmentChunks.", validList, h.chunks(), Object::equals); Assert.assertEquals("Unexpected length.", 15, h.length()); new SegmentChunk("s3", h.length() - 1), new SegmentChunk("s4", h.length() + 1)); secondBadList.get(0).setLength(2); AssertExtensions.assertThrows(
Preconditions.checkState(source.isSealed(), "Cannot concat segment '%s' into '%s' because it is not sealed.", sourceSegment, target.getSegmentName()); if (source.length() == 0) { List<SegmentChunk> newSegmentChunks = rebase(source.chunks(), target.length()); sealActiveChunk(target); serializeBeginConcat(target, source);
/** * Tests the ability to concat using the header file for those cases when native concat cannot be used because the * source Segment has multiple SegmentChunks. */ @Test public void testConcatHeaderMultiFile() throws Exception { final int initialTargetLength = (int) DEFAULT_ROLLING_POLICY.getMaxLength() / 2; final String sourceSegmentName = "SourceSegment"; @Cleanup val baseStorage = new InMemoryStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); // Create a Target Segment and a Source Segment and write some data to them. s.create(SEGMENT_NAME); val targetHandle = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME); val writeStream = new ByteArrayOutputStream(); populate(s, targetHandle, 1, initialTargetLength, initialTargetLength, writeStream); s.create(sourceSegmentName); val sourceHandle = (RollingSegmentHandle) s.openWrite(sourceSegmentName); populate(s, sourceHandle, APPENDS_PER_SEGMENT, initialTargetLength, initialTargetLength, writeStream); s.seal(sourceHandle); // Concat and verify the handle has been updated accordingly. s.concat(targetHandle, initialTargetLength, sourceSegmentName); checkConcatResult(s, targetHandle, sourceSegmentName, 1 + sourceHandle.chunks().size(), initialTargetLength + (int) sourceHandle.length()); checkWrittenData(writeStream.toByteArray(), s.openRead(SEGMENT_NAME), s); }
checkConcatResult(s, targetHandle, sourceSegmentName, 1 + sourceHandle.chunks().size(), initialTargetLength + (int) sourceHandle.length()); checkWrittenData(writeStream.toByteArray(), s.openRead(SEGMENT_NAME), s);
if (writeHandle.isSealed() && segmentChunk.getLastOffset() == writeHandle.length()) { expectedExists = true; } else {
chunk.setLength(123L); Assert.assertEquals("Unexpected value for length() after adding one SegmentChunk.", chunk.getStartOffset() + chunk.getLength(), h.length()); AssertExtensions.assertListEquals("Unexpected contents for chunks().", Collections.singletonList(chunk), h.chunks(), Object::equals); Assert.assertEquals("Unexpected number of registered SegmentChunks.", 2, h.chunks().size()); Assert.assertEquals("Unexpected value for length() after adding two SegmentChunk.", chunk2.getStartOffset() + chunk2.getLength(), h.length()); Assert.assertEquals("Unexpected lastChunk.", chunk2, h.lastChunk()); h.lastChunk().markInexistent();
Assert.assertNull("Unexpected lastChunk for empty handle.", h.lastChunk()); Assert.assertEquals("Unexpected contents in chunks() for empty handle.", 0, h.chunks().size()); Assert.assertEquals("Unexpected value for length() for empty handle.", 0, h.length()); Assert.assertEquals("Unexpected value for getHeaderLength() for empty handle.", 0, h.getHeaderLength()); Assert.assertEquals("Unexpected value for getHeaderHandle().", headerHandle, h.getHeaderHandle());