private void ensureNotDeleted(RollingSegmentHandle handle) throws StreamSegmentNotExistsException { if (handle.isDeleted()) { throw new StreamSegmentNotExistsException(handle.getSegmentName()); } }
private void ensureNotSealed(RollingSegmentHandle handle) throws StreamSegmentSealedException { if (handle.isSealed()) { throw new StreamSegmentSealedException(handle.getSegmentName()); } }
private void updateHandle(RollingSegmentHandle handle, byte[] data) throws StreamSegmentException { try { this.baseStorage.write(handle.getHeaderHandle(), handle.getHeaderLength(), new ByteArrayInputStream(data), data.length); handle.increaseHeaderLength(data.length); log.debug("Header for '{}' updated with {} bytes for a length of {}.", handle.getSegmentName(), data.length, handle.getHeaderLength()); } catch (BadOffsetException ex) { // If we get BadOffsetException when writing the Handle, it means it was modified externally. throw new StorageNotPrimaryException(handle.getSegmentName(), ex); } }
private void ensureOffset(RollingSegmentHandle handle, long offset) throws StreamSegmentException { if (offset != handle.length()) { // Force-refresh the handle to make sure it is still in sync with reality. Make sure we open a read handle // so that we don't force any sort of fencing during this process. val refreshedHandle = openHandle(handle.getSegmentName(), true); handle.refresh(refreshedHandle); log.debug("Handle refreshed: {}.", handle); if (offset != handle.length()) { // Still in disagreement; throw exception. throw new BadOffsetException(handle.getSegmentName(), handle.length(), offset); } } }
/** * Updates the contents of this handle with information from the given one. * * @param source The RollingSegmentHandle to update from. */ synchronized void refresh(RollingSegmentHandle source) { Preconditions.checkArgument(source.getSegmentName().equals(this.getSegmentName()), "SegmentName mismatch."); if (this.readOnly == source.readOnly) { // Update the header handle, but only if both this handle and the source one have the same read-only flag. // Otherwise we risk attaching a read-only header handle to a read-write handle or vice-versa. this.headerHandle = source.headerHandle; } this.segmentChunks = new ArrayList<>(source.chunks()); setHeaderLength(source.getHeaderLength()); if (source.isSealed()) { markSealed(); } if (source.isDeleted()) { markDeleted(); } }
@SneakyThrows(StreamingException.class) private void checkTruncatedSegment(StreamingException ex, RollingSegmentHandle handle, SegmentChunk segmentChunk) { if (ex != null && (Exceptions.unwrap(ex) instanceof StreamSegmentNotExistsException) || !segmentChunk.exists()) { // We ran into a SegmentChunk that does not exist (either marked as such or due to a failed read). segmentChunk.markInexistent(); String message = String.format("Offsets %d-%d have been deleted.", segmentChunk.getStartOffset(), segmentChunk.getLastOffset()); ex = new StreamSegmentTruncatedException(handle.getSegmentName(), message, ex); } if (ex != null) { throw ex; } }
private void rollover(RollingSegmentHandle handle) throws StreamSegmentException { Preconditions.checkArgument(handle.getHeaderHandle() != null, "Cannot rollover a Segment with no header."); Preconditions.checkArgument(!handle.isReadOnly(), "Cannot rollover using a read-only handle."); Preconditions.checkArgument(!handle.isSealed(), "Cannot rollover a Sealed Segment."); log.debug("Rolling over '{}'.", handle); sealActiveChunk(handle); try { createChunk(handle); } catch (StreamSegmentExistsException ex) { // It may be possible that a concurrent rollover request using a different handle (either from this instance // or another) has already created the new chunk. Refresh the handle and try again. This is usually the case // with concurrent Storage instances trying to modify the same segment at the same time. int chunkCount = handle.chunks().size(); handle.refresh(openHandle(handle.getSegmentName(), false)); if (chunkCount == handle.chunks().size()) { // Nothing changed; re-throw the exception. throw ex; } else { // We've just refreshed the handle and picked up the latest chunk. Move on. log.warn("Aborted rollover due to concurrent rollover detected ('{}').", handle); } } }
private void serializeHandle(RollingSegmentHandle handle) throws StreamSegmentException { ByteArraySegment handleData = HandleSerializer.serialize(handle); try { this.baseStorage.write(handle.getHeaderHandle(), 0, handleData.getReader(), handleData.getLength()); handle.setHeaderLength(handleData.getLength()); log.debug("Header for '{}' fully serialized to '{}'.", handle.getSegmentName(), handle.getHeaderHandle().getSegmentName()); } catch (BadOffsetException ex) { // If we get BadOffsetException when writing the Handle, it means it was modified externally. throw new StorageNotPrimaryException(handle.getSegmentName(), ex); } }
private void sealActiveChunk(RollingSegmentHandle handle) throws StreamSegmentException { SegmentHandle activeChunk = handle.getActiveChunkHandle(); SegmentChunk last = handle.lastChunk(); if (activeChunk != null && !last.isSealed()) { this.baseStorage.seal(activeChunk); handle.setActiveChunkHandle(null); last.markSealed(); log.debug("Sealed active SegmentChunk '{}' for '{}'.", activeChunk.getSegmentName(), handle.getSegmentName()); } }
@Override public void seal(SegmentHandle handle) throws StreamSegmentException { val h = asWritableHandle(handle); ensureNotDeleted(h); long traceId = LoggerHelpers.traceEnter(log, "seal", handle); sealActiveChunk(h); SegmentHandle headerHandle = h.getHeaderHandle(); if (headerHandle != null) { this.baseStorage.seal(headerHandle); } h.markSealed(); log.debug("Sealed Header for '{}'.", h.getSegmentName()); LoggerHelpers.traceLeave(log, "seal", traceId, handle); }
private void createHeader(RollingSegmentHandle handle) throws StreamSegmentException { Preconditions.checkArgument(handle.getHeaderHandle() == null, "handle already has a header."); // Create a new Header SegmentChunk. String headerName = StreamSegmentNameUtils.getHeaderSegmentName(handle.getSegmentName()); this.baseStorage.create(headerName); val headerHandle = this.baseStorage.openWrite(headerName); // Create a new Handle and serialize it, after which update the original handle. val newHandle = new RollingSegmentHandle(headerHandle, handle.getRollingPolicy(), handle.chunks()); serializeHandle(newHandle); handle.refresh(newHandle); }
private void createChunk(RollingSegmentHandle handle) throws StreamSegmentException { // Create new active SegmentChunk, only after which serialize the handle update and update the handle. // We ignore if the SegmentChunk exists and is empty - that's most likely due to a previous failed attempt. long segmentLength = handle.length(); SegmentChunk newSegmentChunk = SegmentChunk.forSegment(handle.getSegmentName(), segmentLength); try { this.baseStorage.create(newSegmentChunk.getName()); } catch (StreamSegmentExistsException ex) { checkIfEmptyAndNotSealed(ex, newSegmentChunk.getName()); } serializeNewChunk(handle, newSegmentChunk); val activeHandle = this.baseStorage.openWrite(newSegmentChunk.getName()); handle.addChunk(newSegmentChunk, activeHandle); log.debug("Created new SegmentChunk '{}' for '{}'.", newSegmentChunk, handle); }
@Override public SegmentProperties getStreamSegmentInfo(String segmentName) throws StreamSegmentException { val handle = (RollingSegmentHandle) openRead(segmentName); return StreamSegmentInformation .builder() .name(handle.getSegmentName()) .sealed(handle.isSealed()) .length(handle.length()) .build(); }
private void unsealLastChunkIfNecessary(RollingSegmentHandle handle) throws StreamSegmentException { SegmentChunk last = handle.lastChunk(); if (last == null || !last.isSealed()) { // Nothing to do. return; } SegmentHandle activeChunk = handle.getActiveChunkHandle(); boolean needsHandleUpdate = activeChunk == null; if (needsHandleUpdate) { // We didn't have a pointer to the active chunk's Handle because the chunk was sealed before open-write. activeChunk = this.baseStorage.openWrite(last.getName()); } try { this.baseStorage.unseal(activeChunk); } catch (UnsupportedOperationException e) { log.warn("Unable to unseal SegmentChunk '{}' since base storage does not support unsealing.", last); return; } last.markUnsealed(); if (needsHandleUpdate) { activeChunk = this.baseStorage.openWrite(last.getName()); handle.setActiveChunkHandle(activeChunk); } log.debug("Unsealed active SegmentChunk '{}' for '{}'.", activeChunk.getSegmentName(), handle.getSegmentName()); }
private void assertHandleEquals(RollingSegmentHandle expected, RollingSegmentHandle actual, SegmentHandle headerHandle) { Assert.assertEquals("getSegmentName", expected.getSegmentName(), actual.getSegmentName()); AssertExtensions.assertListEquals("chunks", expected.chunks(), actual.chunks(), this::chunkEquals); Assert.assertEquals("getRollingPolicy", expected.getRollingPolicy().getMaxLength(), actual.getRollingPolicy().getMaxLength()); Assert.assertEquals("getHeaderHandle", headerHandle, actual.getHeaderHandle()); Assert.assertEquals("isReadOnly", headerHandle.isReadOnly(), expected.isReadOnly()); }
sourceSegment, target.getSegmentName()); if (source.length() == 0) { "Cannot use Segment '%s' as concat source because it is truncated.", source.getSegmentName());
val chunk = new SegmentChunk(StreamSegmentNameUtils.getSegmentChunkName(source.getSegmentName(), source.length()), source.length()); chunk.setLength(i + 1); source.addChunks(Collections.singletonList(chunk));
Assert.assertEquals("Unexpected value for getHeaderLength() for empty handle.", 0, h.getHeaderLength()); Assert.assertEquals("Unexpected value for getHeaderHandle().", headerHandle, h.getHeaderHandle()); Assert.assertEquals("Unexpected segment name.", SEGMENT_NAME, h.getSegmentName()); Assert.assertEquals("Unexpected rolling policy.", DEFAULT_ROLLING_POLICY, h.getRollingPolicy()); Assert.assertTrue("Unexpected value for isReadOnly.", h.isReadOnly());