private void serializeBeginConcat(RollingSegmentHandle targetHandle, RollingSegmentHandle sourceHandle) throws StreamSegmentException { byte[] updateData = HandleSerializer.serializeConcat(sourceHandle.chunks().size(), targetHandle.length()); updateHandle(targetHandle, updateData); }
private void rollover(RollingSegmentHandle handle) throws StreamSegmentException { Preconditions.checkArgument(handle.getHeaderHandle() != null, "Cannot rollover a Segment with no header."); Preconditions.checkArgument(!handle.isReadOnly(), "Cannot rollover using a read-only handle."); Preconditions.checkArgument(!handle.isSealed(), "Cannot rollover a Sealed Segment."); log.debug("Rolling over '{}'.", handle); sealActiveChunk(handle); try { createChunk(handle); } catch (StreamSegmentExistsException ex) { // It may be possible that a concurrent rollover request using a different handle (either from this instance // or another) has already created the new chunk. Refresh the handle and try again. This is usually the case // with concurrent Storage instances trying to modify the same segment at the same time. int chunkCount = handle.chunks().size(); handle.refresh(openHandle(handle.getSegmentName(), false)); if (chunkCount == handle.chunks().size()) { // Nothing changed; re-throw the exception. throw ex; } else { // We've just refreshed the handle and picked up the latest chunk. Move on. log.warn("Aborted rollover due to concurrent rollover detected ('{}').", handle); } } }
private void refreshChunkExistence(RollingSegmentHandle handle) { // We check all SegmentChunks that we assume exist for actual existence (since once deleted, they can't come back). for (SegmentChunk s : handle.chunks()) { if (s.exists() && !this.baseStorage.exists(s.getName())) { s.markInexistent(); } } }
private void deleteChunks(RollingSegmentHandle handle, Predicate<SegmentChunk> canDelete) throws StreamSegmentException { for (SegmentChunk s : handle.chunks()) { if (s.exists() && canDelete.test(s)) { try { val subHandle = this.baseStorage.openWrite(s.getName()); this.baseStorage.delete(subHandle); s.markInexistent(); log.debug("Deleted SegmentChunk '{}' for '{}'.", s, handle); } catch (StreamSegmentNotExistsException ex) { // Ignore; It's OK if it doesn't exist; just make sure the handle is updated. s.markInexistent(); } } } }
/** * Serializes an entire RollingSegmentHandle into a new ByteArraySegment. * * @param handle The RollingSegmentHandle to serialize. * @return A ByteArraySegment with the serialization. */ @SneakyThrows(IOException.class) static ByteArraySegment serialize(RollingSegmentHandle handle) { try (EnhancedByteArrayOutputStream os = new EnhancedByteArrayOutputStream()) { //1. Policy Max Size. os.write(combine(KEY_POLICY_MAX_SIZE, Long.toString(handle.getRollingPolicy().getMaxLength()))); //2. Chunks. handle.chunks().forEach(chunk -> os.write(serializeChunk(chunk))); return os.getData(); } }
/** * Updates the contents of this handle with information from the given one. * * @param source The RollingSegmentHandle to update from. */ synchronized void refresh(RollingSegmentHandle source) { Preconditions.checkArgument(source.getSegmentName().equals(this.getSegmentName()), "SegmentName mismatch."); if (this.readOnly == source.readOnly) { // Update the header handle, but only if both this handle and the source one have the same read-only flag. // Otherwise we risk attaching a read-only header handle to a read-write handle or vice-versa. this.headerHandle = source.headerHandle; } this.segmentChunks = new ArrayList<>(source.chunks()); setHeaderLength(source.getHeaderLength()); if (source.isSealed()) { markSealed(); } if (source.isDeleted()) { markDeleted(); } }
private void createHeader(RollingSegmentHandle handle) throws StreamSegmentException { Preconditions.checkArgument(handle.getHeaderHandle() == null, "handle already has a header."); // Create a new Header SegmentChunk. String headerName = StreamSegmentNameUtils.getHeaderSegmentName(handle.getSegmentName()); this.baseStorage.create(headerName); val headerHandle = this.baseStorage.openWrite(headerName); // Create a new Handle and serialize it, after which update the original handle. val newHandle = new RollingSegmentHandle(headerHandle, handle.getRollingPolicy(), handle.chunks()); serializeHandle(newHandle); handle.refresh(newHandle); }
private void checkConcatResult(RollingStorage s, RollingSegmentHandle targetHandle, String sourceSegmentName, int expectedChunkCount, int expectedLength) throws Exception { Assert.assertFalse("Expecting the source segment to not exist anymore.", s.exists(sourceSegmentName)); Assert.assertEquals("Unexpected number of SegmentChunks in target.", expectedChunkCount, targetHandle.chunks().size()); Assert.assertEquals("Unexpected target length.", expectedLength, targetHandle.length()); // Reload the handle and verify nothing strange happened in Storage. val targetHandle2 = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME); Assert.assertEquals("Unexpected number of SegmentChunks in reloaded target handle.", expectedChunkCount, targetHandle2.chunks().size()); Assert.assertEquals("Unexpected reloaded target length.", targetHandle.length(), targetHandle2.length()); }
@Test public void testWriteAfterHeaderMerge() throws Exception { final String segmentName = "Segment"; final int writeLength = 21; @Cleanup val s = createStorage(); s.initialize(1); // Create Target Segment with infinite rolling. Do not write anything to it yet. val writeHandle = s.create(segmentName, SegmentRollingPolicy.NO_ROLLING, TIMEOUT) .thenCompose(v -> s.openWrite(segmentName)).join(); final Random rnd = new Random(0); byte[] writeBuffer = new byte[writeLength]; val writeStream = new ByteArrayOutputStream(); // Create a source segment, write a little bit to it, then seal & merge it. String sourceSegment = segmentName + "_Source"; val sourceHandle = s.create(sourceSegment, TIMEOUT).thenCompose(v -> s.openWrite(sourceSegment)).join(); rnd.nextBytes(writeBuffer); s.write(sourceHandle, 0, new ByteArrayInputStream(writeBuffer), writeBuffer.length, TIMEOUT).join(); s.seal(sourceHandle, TIMEOUT).join(); s.concat(writeHandle, writeStream.size(), sourceSegment, TIMEOUT).join(); writeStream.write(writeBuffer); // Write directly to the target segment. rnd.nextBytes(writeBuffer); s.write(writeHandle, writeStream.size(), new ByteArrayInputStream(writeBuffer), writeBuffer.length, TIMEOUT).join(); writeStream.write(writeBuffer); // Get a read handle, which will also fetch the number of chunks for us. val readHandle = (RollingSegmentHandle) s.openRead(segmentName).join(); Assert.assertEquals("Unexpected number of chunks created.", 1, readHandle.chunks().size()); }
String failOnDelete = writeHandle.chunks().get(failAtIndex).getName(); baseStorage.deleteFailure = sn -> sn.equals(failOnDelete) ? new IntentionalException() : null; AssertExtensions.assertThrows( Assert.assertFalse("Expected first SegmentChunk to be marked as deleted.", writeHandle.chunks().get(failAtIndex - 1).exists()); Assert.assertTrue("Expected failed-to-delete SegmentChunk to not be marked as deleted.", writeHandle.chunks().get(failAtIndex).exists()); Assert.assertTrue("Expected subsequent SegmentChunk to not be marked as deleted.", writeHandle.chunks().get(failAtIndex + 1).exists()); Assert.assertFalse("Expecting the segment to be deleted.", s.exists(SEGMENT_NAME)); Assert.assertTrue("Expected the handle to be marked as deleted.", writeHandle.isDeleted()); Assert.assertFalse("Expected all SegmentChunks to be marked as deleted.", writeHandle.chunks().stream().anyMatch(SegmentChunk::exists));
() -> h.addChunks(firstBadList), ex -> ex instanceof IllegalArgumentException); Assert.assertEquals("Not expecting any SegmentChunks to be added.", 0, h.chunks().size()); Assert.assertEquals("Unexpected length().", 0, h.length()); validList.get(1).setLength(5); h.addChunks(validList); AssertExtensions.assertListEquals("Unexpected list of SegmentChunks.", validList, h.chunks(), Object::equals); Assert.assertEquals("Unexpected length.", 15, h.length());
private void assertHandleEquals(RollingSegmentHandle expected, RollingSegmentHandle actual, SegmentHandle headerHandle) { Assert.assertEquals("getSegmentName", expected.getSegmentName(), actual.getSegmentName()); AssertExtensions.assertListEquals("chunks", expected.chunks(), actual.chunks(), this::chunkEquals); Assert.assertEquals("getRollingPolicy", expected.getRollingPolicy().getMaxLength(), actual.getRollingPolicy().getMaxLength()); Assert.assertEquals("getHeaderHandle", headerHandle, actual.getHeaderHandle()); Assert.assertEquals("isReadOnly", headerHandle.isReadOnly(), expected.isReadOnly()); }
/** * Tests the ability of the Handle to refresh based on information from another similar handle. */ @Test public void testRefresh() { val headerHandle = new TestHandle(HEADER_NAME, true); val target = new RollingSegmentHandle(headerHandle, DEFAULT_ROLLING_POLICY, Collections.singletonList(new SegmentChunk("s1", 0L))); val source = new RollingSegmentHandle(headerHandle, DEFAULT_ROLLING_POLICY, Arrays.asList( new SegmentChunk("s1", 0L), new SegmentChunk("s2", 100L))); source.chunks().get(0).setLength(100); source.markSealed(); source.setHeaderLength(1000); source.setActiveChunkHandle(new TestHandle(source.lastChunk().getName(), false)); target.refresh(source); Assert.assertEquals("Unexpected getHeaderLength()", source.getHeaderLength(), target.getHeaderLength()); AssertExtensions.assertListEquals("Unexpected chunks()", source.chunks(), target.chunks(), Object::equals); Assert.assertTrue("Unexpected isSealed.", target.isSealed()); Assert.assertNull("Not expecting any ActiveSegmentHandle to be copied.", target.getActiveChunkHandle()); }
source.addChunks(concatHandle.chunks().stream() .map(s -> s.withNewOffset(s.getStartOffset() + source.length())).collect(Collectors.toList()));
/** * Tests the ability to concat using the header file for those cases when native concat cannot be used because the * source Segment has multiple SegmentChunks. */ @Test public void testConcatHeaderMultiFile() throws Exception { final int initialTargetLength = (int) DEFAULT_ROLLING_POLICY.getMaxLength() / 2; final String sourceSegmentName = "SourceSegment"; @Cleanup val baseStorage = new InMemoryStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); // Create a Target Segment and a Source Segment and write some data to them. s.create(SEGMENT_NAME); val targetHandle = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME); val writeStream = new ByteArrayOutputStream(); populate(s, targetHandle, 1, initialTargetLength, initialTargetLength, writeStream); s.create(sourceSegmentName); val sourceHandle = (RollingSegmentHandle) s.openWrite(sourceSegmentName); populate(s, sourceHandle, APPENDS_PER_SEGMENT, initialTargetLength, initialTargetLength, writeStream); s.seal(sourceHandle); // Concat and verify the handle has been updated accordingly. s.concat(targetHandle, initialTargetLength, sourceSegmentName); checkConcatResult(s, targetHandle, sourceSegmentName, 1 + sourceHandle.chunks().size(), initialTargetLength + (int) sourceHandle.length()); checkWrittenData(writeStream.toByteArray(), s.openRead(SEGMENT_NAME), s); }
checkConcatResult(s, targetHandle, sourceSegmentName, 1 + sourceHandle.chunks().size(), initialTargetLength + (int) sourceHandle.length()); checkWrittenData(writeStream.toByteArray(), s.openRead(SEGMENT_NAME), s);
() -> h.addChunk(new SegmentChunk("s", 0L), new TestHandle("s2", false)), ex -> ex instanceof IllegalArgumentException); Assert.assertEquals("Not expecting any SegmentChunks to be added.", 0, h.chunks().size()); Assert.assertNull("Not expecting the Active SegmentChunk handle to be set.", h.getActiveChunkHandle()); chunk.getStartOffset() + chunk.getLength(), h.length()); AssertExtensions.assertListEquals("Unexpected contents for chunks().", Collections.singletonList(chunk), h.chunks(), Object::equals); Assert.assertEquals("Unexpected lastChunk.", chunk, h.lastChunk()); chunk2.setLength(234L); h.addChunk(chunk2, new TestHandle("s2", false)); Assert.assertEquals("Unexpected number of registered SegmentChunks.", 2, h.chunks().size()); Assert.assertEquals("Unexpected value for length() after adding two SegmentChunk.", chunk2.getStartOffset() + chunk2.getLength(), h.length());
for (SegmentChunk s : handle.chunks()) { if (last != null) { last.setLength(s.getStartOffset() - last.getStartOffset());
for (SegmentChunk segmentChunk : writeHandle.chunks()) { boolean expectedExists; if (writeHandle.isSealed() && segmentChunk.getLastOffset() == writeHandle.length()) {
val h = new RollingSegmentHandle(headerHandle, DEFAULT_ROLLING_POLICY, new ArrayList<>()); Assert.assertNull("Unexpected lastChunk for empty handle.", h.lastChunk()); Assert.assertEquals("Unexpected contents in chunks() for empty handle.", 0, h.chunks().size()); Assert.assertEquals("Unexpected value for length() for empty handle.", 0, h.length()); Assert.assertEquals("Unexpected value for getHeaderLength() for empty handle.", 0, h.getHeaderLength());