/** * Creates a new instance of the SegmentChunk class with the same information as this one, but with a new offset. * * @param newOffset The new offset. * @return A new SegmentChunk. */ SegmentChunk withNewOffset(long newOffset) { SegmentChunk ns = new SegmentChunk(this.name, newOffset); ns.setLength(getLength()); if (isSealed()) { ns.markSealed(); } if (!exists()) { ns.markInexistent(); } return ns; }
@SneakyThrows(StreamingException.class) private void checkTruncatedSegment(StreamingException ex, RollingSegmentHandle handle, SegmentChunk segmentChunk) { if (ex != null && (Exceptions.unwrap(ex) instanceof StreamSegmentNotExistsException) || !segmentChunk.exists()) { // We ran into a SegmentChunk that does not exist (either marked as such or due to a failed read). segmentChunk.markInexistent(); String message = String.format("Offsets %d-%d have been deleted.", segmentChunk.getStartOffset(), segmentChunk.getLastOffset()); ex = new StreamSegmentTruncatedException(handle.getSegmentName(), message, ex); } if (ex != null) { throw ex; } }
private void refreshChunkExistence(RollingSegmentHandle handle) { // We check all SegmentChunks that we assume exist for actual existence (since once deleted, they can't come back). for (SegmentChunk s : handle.chunks()) { if (s.exists() && !this.baseStorage.exists(s.getName())) { s.markInexistent(); } } }
private boolean canTruncate(SegmentChunk segmentChunk, long truncationOffset) { // We should only truncate those SegmentChunks that are entirely before the truncationOffset. An empty SegmentChunk // that starts exactly at the truncationOffset should be spared (this means we truncate the entire Segment), as // we need that SegmentChunk to determine the actual length of the Segment. return segmentChunk.getStartOffset() < truncationOffset && segmentChunk.getLastOffset() <= truncationOffset; }
private List<SegmentChunk> rebase(List<SegmentChunk> segmentChunks, long newStartOffset) { AtomicLong segmentOffset = new AtomicLong(newStartOffset); return segmentChunks.stream() .map(s -> s.withNewOffset(segmentOffset.getAndAdd(s.getLength()))) .collect(Collectors.toList()); }
() -> h.addChunk(new SegmentChunk("s", 0L), null), ex -> ex instanceof NullPointerException); AssertExtensions.assertThrows( "addChunk allowed adding a read-only ActiveSegmentHandle.", () -> h.addChunk(new SegmentChunk("s", 0L), new TestHandle("s", true)), ex -> ex instanceof IllegalArgumentException); AssertExtensions.assertThrows( "addChunk allowed adding an ActiveSegmentHandle with different name..", () -> h.addChunk(new SegmentChunk("s", 0L), new TestHandle("s2", false)), ex -> ex instanceof IllegalArgumentException); Assert.assertEquals("Not expecting any SegmentChunks to be added.", 0, h.chunks().size()); Assert.assertNull("Not expecting the Active SegmentChunk handle to be set.", h.getActiveChunkHandle()); val chunk = new SegmentChunk("s1", 100L); h.addChunk(chunk, new TestHandle("s1", false)); chunk.setLength(123L); Assert.assertEquals("Unexpected value for length() after adding one SegmentChunk.", chunk.getStartOffset() + chunk.getLength(), h.length()); AssertExtensions.assertListEquals("Unexpected contents for chunks().", Collections.singletonList(chunk), h.chunks(), Object::equals); () -> h.addChunk(new SegmentChunk("s2", chunk.getLastOffset() + 1), new TestHandle("s2", false)), ex -> ex instanceof IllegalArgumentException); chunk.markInexistent(); val chunk2 = new SegmentChunk("s2", chunk.getLastOffset()); chunk2.setLength(234L); h.addChunk(chunk2, new TestHandle("s2", false)); Assert.assertEquals("Unexpected number of registered SegmentChunks.", 2, h.chunks().size());
for (SegmentChunk s : handle.chunks()) { if (last != null) { last.setLength(s.getStartOffset() - last.getStartOffset()); last.markSealed(); val si = this.baseStorage.getStreamSegmentInfo(last.getName()); last.setLength(si.getLength()); if (si.isSealed()) { last.markSealed(); if (handle.getHeaderHandle() == null) { handle.markSealed();
private RollingSegmentHandle newHandle(String segmentName, int chunkCount) { val chunks = new ArrayList<SegmentChunk>(); long offset = 0; val rnd = new Random(0); for (int i = 0; i < chunkCount; i++) { val chunk = new SegmentChunk(StreamSegmentNameUtils.getSegmentChunkName(segmentName, offset), offset); chunk.setLength(MathHelpers.abs(rnd.nextInt())); if (i < chunkCount - 1) { chunk.markSealed(); } chunks.add(chunk); offset += chunk.getLength(); } return new RollingSegmentHandle(new TestHandle(StreamSegmentNameUtils.getHeaderSegmentName(segmentName), false), TEST_ROLLING_POLICY, chunks); }
int currentIndex = CollectionHelpers.binarySearch(chunks, s -> offset < s.getStartOffset() ? -1 : (offset >= s.getLastOffset() ? 1 : 0)); assert currentIndex >= 0 : "unable to locate first SegmentChunk index."; if (current.getLength() == 0) { long readOffset = offset + bytesRead - current.getStartOffset(); int readLength = (int) Math.min(length - bytesRead, current.getLength() - readOffset); assert readOffset >= 0 && readLength >= 0 : "negative readOffset or readLength"; val sh = this.baseStorage.openRead(current.getName()); int count = this.baseStorage.read(sh, readOffset, buffer, bufferOffset + bytesRead, readLength); bytesRead += count; if (readOffset + count >= current.getLength()) { currentIndex++;
if (writeHandle.isSealed() && segmentChunk.getLastOffset() == writeHandle.length()) { expectedExists = true; } else { expectedExists = segmentChunk.getLastOffset() > truncateOffset || (segmentChunk.getStartOffset() == segmentChunk.getLastOffset() && segmentChunk.getLastOffset() == truncateOffset); expectedExists, segmentChunk.exists()); boolean existsInStorage = baseStorage.exists(segmentChunk.getName()); Assert.assertEquals("Expected SegmentChunk deletion status for " + segmentChunk + ", truncation offset = " + truncateOffset, expectedExists, existsInStorage); AssertExtensions.assertThrows( "Not expecting a read from a truncated SegmentChunk to work.", () -> s.read(readHandle, segmentChunk.getLastOffset() - 1, new byte[1], 0, 1), ex -> ex instanceof StreamSegmentTruncatedException);
os.write(serialize(concatHandle)); source.addChunks(concatHandle.chunks().stream() .map(s -> s.withNewOffset(s.getStartOffset() + source.length())).collect(Collectors.toList())); val chunk = new SegmentChunk(StreamSegmentNameUtils.getSegmentChunkName(source.getSegmentName(), source.length()), source.length()); chunk.setLength(i + 1); source.addChunks(Collections.singletonList(chunk)); os.write(HandleSerializer.serializeChunk(chunk));
/** * Adds multiple SegmentChunks. * * @param segmentChunks The SegmentChunks to add. These SegmentChunks must be in continuity of any existing SegmentChunks. */ synchronized void addChunks(List<SegmentChunk> segmentChunks) { Preconditions.checkState(!this.sealed, "Cannot add SegmentChunks for a Sealed Handle."); long expectedOffset = 0; if (this.segmentChunks.size() > 0) { expectedOffset = this.segmentChunks.get(this.segmentChunks.size() - 1).getLastOffset(); } else if (segmentChunks.size() > 0) { expectedOffset = segmentChunks.get(0).getStartOffset(); } for (SegmentChunk s : segmentChunks) { Preconditions.checkArgument(s.getStartOffset() == expectedOffset, "Invalid SegmentChunk StartOffset. Expected %s, given %s.", expectedOffset, s.getStartOffset()); expectedOffset += s.getLength(); } this.segmentChunks.addAll(segmentChunks); this.activeChunkHandle = null; }
/** * Adds a new SegmentChunk. * * @param segmentChunk The SegmentChunk to add. This SegmentChunk must be in continuity of any existing SegmentChunks. * @param activeChunkHandle The newly added SegmentChunk's write handle. */ synchronized void addChunk(SegmentChunk segmentChunk, SegmentHandle activeChunkHandle) { Preconditions.checkState(!this.sealed, "Cannot add SegmentChunks for a Sealed Handle."); if (this.segmentChunks.size() > 0) { long expectedOffset = this.segmentChunks.get(this.segmentChunks.size() - 1).getLastOffset(); Preconditions.checkArgument(segmentChunk.getStartOffset() == expectedOffset, "Invalid SegmentChunk StartOffset. Expected %s, given %s.", expectedOffset, segmentChunk.getStartOffset()); } // Update the SegmentChunk and its Handle atomically. Preconditions.checkNotNull(activeChunkHandle, "activeChunkHandle"); Preconditions.checkArgument(!activeChunkHandle.isReadOnly(), "Active SegmentChunk handle cannot be readonly."); Preconditions.checkArgument(activeChunkHandle.getSegmentName().equals(segmentChunk.getName()), "Active SegmentChunk handle must be for the last SegmentChunk."); this.activeChunkHandle = activeChunkHandle; this.segmentChunks.add(segmentChunk); }
/** * Tests the ability of the Handle to refresh based on information from another similar handle. */ @Test public void testRefresh() { val headerHandle = new TestHandle(HEADER_NAME, true); val target = new RollingSegmentHandle(headerHandle, DEFAULT_ROLLING_POLICY, Collections.singletonList(new SegmentChunk("s1", 0L))); val source = new RollingSegmentHandle(headerHandle, DEFAULT_ROLLING_POLICY, Arrays.asList( new SegmentChunk("s1", 0L), new SegmentChunk("s2", 100L))); source.chunks().get(0).setLength(100); source.markSealed(); source.setHeaderLength(1000); source.setActiveChunkHandle(new TestHandle(source.lastChunk().getName(), false)); target.refresh(source); Assert.assertEquals("Unexpected getHeaderLength()", source.getHeaderLength(), target.getHeaderLength()); AssertExtensions.assertListEquals("Unexpected chunks()", source.chunks(), target.chunks(), Object::equals); Assert.assertTrue("Unexpected isSealed.", target.isSealed()); Assert.assertNull("Not expecting any ActiveSegmentHandle to be copied.", target.getActiveChunkHandle()); }
if (lastTarget == null || lastTarget.isSealed()) { this.baseStorage.concat(target.getActiveChunkHandle(), target.lastChunk().getLength(), lastSource.getName()); target.lastChunk().increaseLength(lastSource.getLength()); if (source.getHeaderHandle() != null) { try {
private boolean shouldConcatNatively(RollingSegmentHandle source, RollingSegmentHandle target) { if (source.getHeaderHandle() == null) { // Source does not have a Header, hence we cannot do Header concat. return true; } SegmentChunk lastSource = source.lastChunk(); SegmentChunk lastTarget = target.lastChunk(); return lastSource != null && lastSource.getStartOffset() == 0 && lastTarget != null && !lastTarget.isSealed() && lastTarget.getLength() + lastSource.getLength() <= target.getRollingPolicy().getMaxLength(); }
/** * Serializes a single SegmentChunk. * * @param segmentChunk The SegmentChunk to serialize. * @return A byte array containing the serialization. */ static byte[] serializeChunk(SegmentChunk segmentChunk) { return combine(Long.toString(segmentChunk.getStartOffset()), segmentChunk.getName()); }
@Override public void write(SegmentHandle handle, long offset, InputStream data, int length) throws StreamSegmentException { val h = asWritableHandle(handle); ensureNotDeleted(h); ensureNotSealed(h); ensureOffset(h, offset); long traceId = LoggerHelpers.traceEnter(log, "write", handle, offset, length); // We run this in a loop because we may have to split the write over multiple SegmentChunks in order to avoid exceeding // any SegmentChunk's maximum length. int bytesWritten = 0; while (bytesWritten < length) { if (h.getActiveChunkHandle() == null || h.lastChunk().getLength() >= h.getRollingPolicy().getMaxLength()) { rollover(h); } SegmentChunk last = h.lastChunk(); int writeLength = (int) Math.min(length - bytesWritten, h.getRollingPolicy().getMaxLength() - last.getLength()); assert writeLength > 0 : "non-positive write length"; long chunkOffset = offset + bytesWritten - last.getStartOffset(); this.baseStorage.write(h.getActiveChunkHandle(), chunkOffset, data, writeLength); last.increaseLength(writeLength); bytesWritten += writeLength; } LoggerHelpers.traceLeave(log, "write", traceId, handle, offset, bytesWritten); }
SegmentChunk s = new SegmentChunk(entry.getValue(), offset); Preconditions.checkArgument(lastOffset <= s.getStartOffset(), "SegmentChunk Entry '%s' has out-of-order offset (previous=%s).", s, lastOffset); segmentChunks.add(s); lastOffset = s.getStartOffset();
private void unsealLastChunkIfNecessary(RollingSegmentHandle handle) throws StreamSegmentException { SegmentChunk last = handle.lastChunk(); if (last == null || !last.isSealed()) { // Nothing to do. return; } SegmentHandle activeChunk = handle.getActiveChunkHandle(); boolean needsHandleUpdate = activeChunk == null; if (needsHandleUpdate) { // We didn't have a pointer to the active chunk's Handle because the chunk was sealed before open-write. activeChunk = this.baseStorage.openWrite(last.getName()); } try { this.baseStorage.unseal(activeChunk); } catch (UnsupportedOperationException e) { log.warn("Unable to unseal SegmentChunk '{}' since base storage does not support unsealing.", last); return; } last.markUnsealed(); if (needsHandleUpdate) { activeChunk = this.baseStorage.openWrite(last.getName()); handle.setActiveChunkHandle(activeChunk); } log.debug("Unsealed active SegmentChunk '{}' for '{}'.", activeChunk.getSegmentName(), handle.getSegmentName()); }