@Override public Storage createStorageAdapter() { return new AsyncStorageWrapper(new RollingStorage(this.baseStorage), this.executor); }
@Override public Storage createStorageAdapter() { HDFSStorage s = new HDFSStorage(this.config); return new AsyncStorageWrapper(new RollingStorage(s), this.executor); } }
@Override public Storage createStorageAdapter() { FileSystemStorage s = new FileSystemStorage(this.config); return new AsyncStorageWrapper(new RollingStorage(s), this.executor); } }
@Override public Storage createStorageAdapter() { return new WatchableAsyncStorageWrapper(new RollingStorage(this.baseStorage), this.executor); }
@Override public Storage createStorageAdapter() { S3Config s3Config = new S3Config(config.getUrl()) .withIdentity(config.getAccessKey()) .withSecretKey(config.getSecretKey()) .withNamespace(config.getNamespace()); S3JerseyClient client = new S3JerseyClient(s3Config); ExtendedS3Storage s = new ExtendedS3Storage(client, this.config); return new AsyncStorageWrapper(new RollingStorage(s), this.executor); } }
@Override protected Storage createStorage() { return new AsyncStorageWrapper(new RollingStorage(new InMemoryStorage(), DEFAULT_ROLLING_POLICY), executorService()); }
protected Storage wrap(SyncStorage storage) { return new AsyncStorageWrapper(new RollingStorage(storage, new SegmentRollingPolicy(DEFAULT_ROLLING_SIZE)), executorService()); }
/** * Tests the ability to auto-refresh a Write Handle upon offset disagreement. */ @Test public void testRefreshHandleBadOffset() throws Exception { // Write small and large writes, alternatively. @Cleanup val baseStorage = new InMemoryStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); s.create(SEGMENT_NAME); val h1 = s.openWrite(SEGMENT_NAME); val h2 = s.openWrite(SEGMENT_NAME); // Open now, before writing, so we force a refresh. byte[] data = "data".getBytes(); s.write(h1, 0, new ByteArrayInputStream(data), data.length); s.write(h2, data.length, new ByteArrayInputStream(data), data.length); // Check that no file has exceeded its maximum length. byte[] expectedData = new byte[data.length * 2]; System.arraycopy(data, 0, expectedData, 0, data.length); System.arraycopy(data, 0, expectedData, data.length, data.length); checkWrittenData(expectedData, h2, s); }
@Override public Storage createStorageAdapter() { URI uri = URI.create(endpoint); S3Config s3Config = new S3Config(uri); s3Config = s3Config.withIdentity(config.getAccessKey()).withSecretKey(config.getSecretKey()) .withRetryEnabled(false) .withInitialRetryDelay(1) .withProperty("com.sun.jersey.client.property.connectTimeout", 100); S3JerseyClient client = new S3ClientWrapper(s3Config, filesystemS3); return new AsyncStorageWrapper(new RollingStorage(new ExtendedS3Storage(client, config)), this.storageExecutor); } }
TestContext(AttributeIndexConfig config, CachePolicy cachePolicy) { this.memoryStorage = new InMemoryStorage(); this.memoryStorage.initialize(1); this.storage = new TestContext.TestStorage(new RollingStorage(this.memoryStorage, config.getAttributeSegmentRollingPolicy()), executorService()); this.containerMetadata = new MetadataBuilder(CONTAINER_ID).build(); this.cacheFactory = new InMemoryCacheFactory(); this.cacheManager = new TestCacheManager(cachePolicy, executorService()); val factory = new ContainerAttributeIndexFactoryImpl(config, this.cacheFactory, this.cacheManager, executorService()); this.index = factory.createContainerAttributeIndex(this.containerMetadata, this.storage); }
/** * Tests the ability to truncate Sealed Segments. */ @Test public void testTruncateSealed() throws Exception { // Write small and large writes, alternatively. @Cleanup val baseStorage = new TestStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); // Create a Segment, write some data, then seal it. s.create(SEGMENT_NAME); val appendHandle = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME); val writeStream = new ByteArrayOutputStream(); populate(s, appendHandle, writeStream); s.seal(appendHandle); byte[] writtenData = writeStream.toByteArray(); val truncateHandle = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME); Assert.assertTrue("Handle not read-only after sealing.", truncateHandle.isReadOnly()); Assert.assertTrue("Handle not sealed after sealing.", truncateHandle.isSealed()); // Test that truncate works in this scenario. testProgressiveTruncate(truncateHandle, truncateHandle, writtenData, s, baseStorage); }
/** * Tests the ability to concat using the header file for those cases when native concat cannot be used because the * source Segment has multiple SegmentChunks. */ @Test public void testConcatHeaderMultiFile() throws Exception { final int initialTargetLength = (int) DEFAULT_ROLLING_POLICY.getMaxLength() / 2; final String sourceSegmentName = "SourceSegment"; @Cleanup val baseStorage = new InMemoryStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); // Create a Target Segment and a Source Segment and write some data to them. s.create(SEGMENT_NAME); val targetHandle = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME); val writeStream = new ByteArrayOutputStream(); populate(s, targetHandle, 1, initialTargetLength, initialTargetLength, writeStream); s.create(sourceSegmentName); val sourceHandle = (RollingSegmentHandle) s.openWrite(sourceSegmentName); populate(s, sourceHandle, APPENDS_PER_SEGMENT, initialTargetLength, initialTargetLength, writeStream); s.seal(sourceHandle); // Concat and verify the handle has been updated accordingly. s.concat(targetHandle, initialTargetLength, sourceSegmentName); checkConcatResult(s, targetHandle, sourceSegmentName, 1 + sourceHandle.chunks().size(), initialTargetLength + (int) sourceHandle.length()); checkWrittenData(writeStream.toByteArray(), s.openRead(SEGMENT_NAME), s); }
val baseStorage = new TestStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); s.create(SEGMENT_NAME);
/** * Tests the ability to concat using the header file for those cases when native concat cannot be used because the * source Segment has a single SegmentChunk, but it's too large to fit into the Target's active SegmentChunk. */ @Test public void testConcatHeaderSingleFile() throws Exception { final int initialTargetLength = (int) DEFAULT_ROLLING_POLICY.getMaxLength() / 2; final int bigSourceLength = (int) DEFAULT_ROLLING_POLICY.getMaxLength() - initialTargetLength + 1; final String sourceSegmentName = "SourceSegment"; @Cleanup val baseStorage = new InMemoryStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); // Create a Target Segment and a Source Segment and write some data to them. s.create(SEGMENT_NAME); val targetHandle = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME); val writeStream = new ByteArrayOutputStream(); populate(s, targetHandle, 1, initialTargetLength, initialTargetLength, writeStream); s.create(sourceSegmentName); val sourceHandle = (RollingSegmentHandle) s.openWrite(sourceSegmentName); populate(s, sourceHandle, 1, bigSourceLength, bigSourceLength, writeStream); s.seal(sourceHandle); // Concat and verify the handle has been updated accordingly. s.concat(targetHandle, initialTargetLength, sourceSegmentName); checkConcatResult(s, targetHandle, sourceSegmentName, 2, initialTargetLength + bigSourceLength); checkWrittenData(writeStream.toByteArray(), s.openRead(SEGMENT_NAME), s); }
val baseStorage = new TestStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); s.create(SEGMENT_NAME);
/** * Tests the ability to use native concat for those cases when it's appropriate. */ @Test public void testConcatNatively() throws Exception { final int initialTargetLength = (int) DEFAULT_ROLLING_POLICY.getMaxLength() / 2; final int initialSourceLength = (int) DEFAULT_ROLLING_POLICY.getMaxLength() - initialTargetLength; final String sourceSegmentName = "SourceSegment"; @Cleanup val baseStorage = new InMemoryStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); // Create a target Segment and write a little data to it. s.create(SEGMENT_NAME); val targetHandle = (RollingSegmentHandle) s.openWrite(SEGMENT_NAME); val writeStream = new ByteArrayOutputStream(); populate(s, targetHandle, 1, initialTargetLength, initialTargetLength, writeStream); // Create a source Segment and write a little data to it, making sure it is small enough to fit into the target // when we need to concat. s.create(sourceSegmentName); val sourceHandle = (RollingSegmentHandle) s.openWrite(sourceSegmentName); populate(s, sourceHandle, 1, initialSourceLength, initialSourceLength, writeStream); s.seal(sourceHandle); // Concat and verify the handle has been updated accordingly. s.concat(targetHandle, initialTargetLength, sourceSegmentName); checkConcatResult(s, targetHandle, sourceSegmentName, 1, initialTargetLength + initialSourceLength); checkWrittenData(writeStream.toByteArray(), s.openRead(SEGMENT_NAME), s); }
val baseStorage = new InMemoryStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); s.create(SEGMENT_NAME);
val baseStorage = new TestStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1);
val baseStorage = new TestStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1);
/** * Tests the case when Create was interrupted after it created the Header file but before populating it. */ @Test public void testCreateRecovery() throws Exception { @Cleanup val baseStorage = new TestStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); // Create an empty header file. This simulates a create() operation that failed mid-way. baseStorage.create(StreamSegmentNameUtils.getHeaderSegmentName(SEGMENT_NAME)); Assert.assertFalse("Not expecting Segment to exist.", s.exists(SEGMENT_NAME)); AssertExtensions.assertThrows( "Not expecting Segment to exist (getStreamSegmentInfo).", () -> s.getStreamSegmentInfo(SEGMENT_NAME), ex -> ex instanceof StreamSegmentNotExistsException); AssertExtensions.assertThrows( "Not expecting Segment to exist (openHandle).", () -> s.openRead(SEGMENT_NAME), ex -> ex instanceof StreamSegmentNotExistsException); // Retry the operation and verify everything is in place. s.create(SEGMENT_NAME); val si = s.getStreamSegmentInfo(SEGMENT_NAME); Assert.assertEquals("Expected the Segment to have been created.", 0, si.getLength()); }