/** * Tests the append() method. */ @Test public void testAppend() throws Exception { final String segmentName = "segment"; @Cleanup val storage = new InMemoryStorage(); storage.initialize(DEFAULT_EPOCH); storage.create(segmentName); val handle = storage.openWrite(segmentName); ByteArrayOutputStream writeStream = new ByteArrayOutputStream(); for (int j = 0; j < APPENDS_PER_SEGMENT; j++) { byte[] writeData = String.format("Segment_%s_Append_%d", segmentName, j).getBytes(); ByteArrayInputStream dataStream = new ByteArrayInputStream(writeData); storage.append(handle, dataStream, writeData.length); writeStream.write(writeData); } byte[] expectedData = writeStream.toByteArray(); byte[] readBuffer = new byte[expectedData.length]; int bytesRead = storage.read(handle, 0, readBuffer, 0, readBuffer.length); Assert.assertEquals("Unexpected number of bytes read.", readBuffer.length, bytesRead); AssertExtensions.assertArrayEquals("Unexpected read result.", expectedData, 0, readBuffer, 0, bytesRead); }
private void closeInternal() { super.close(); }
private SegmentHandle writeHandle(String segmentName) { return InMemoryStorage.newHandle(segmentName, false); }
/** * Creates a new InMemory Storage, without a rolling wrapper. * * @param executor An Executor to use for async operations. * @return A new InMemoryStorage. */ @VisibleForTesting public static Storage newStorage(Executor executor) { return new AsyncStorageWrapper(new InMemoryStorage(), executor); }
TestContext(AttributeIndexConfig config, CachePolicy cachePolicy) { this.memoryStorage = new InMemoryStorage(); this.memoryStorage.initialize(1); this.storage = new TestContext.TestStorage(new RollingStorage(this.memoryStorage, config.getAttributeSegmentRollingPolicy()), executorService()); this.containerMetadata = new MetadataBuilder(CONTAINER_ID).build(); this.cacheFactory = new InMemoryCacheFactory(); this.cacheManager = new TestCacheManager(cachePolicy, executorService()); val factory = new ContainerAttributeIndexFactoryImpl(config, this.cacheFactory, this.cacheManager, executorService()); this.index = factory.createContainerAttributeIndex(this.containerMetadata, this.storage); }
final String segmentName = "SonHeaderSegment"; @Cleanup val baseStorage = new InMemoryStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); baseStorage.create(segmentName); () -> s.create(segmentName), ex -> ex instanceof StreamSegmentExistsException); Assert.assertTrue("Non-Header Segment does not exist after failed create() attempt.", baseStorage.exists(segmentName)); Assert.assertFalse("A header was left behind (after create).", baseStorage.exists(StreamSegmentNameUtils.getHeaderSegmentName(segmentName))); byte[] writtenData = os.toByteArray(); Assert.assertFalse("A header was left behind (after write).", baseStorage.exists(StreamSegmentNameUtils.getHeaderSegmentName(segmentName))); val baseInfo = baseStorage.getStreamSegmentInfo(segmentName); val rollingInfo = s.getStreamSegmentInfo(segmentName); Assert.assertTrue("Segment not sealed.", baseInfo.isSealed()); baseStorage.create(nonHeaderName); val nonHeaderHandle = s.openWrite(nonHeaderName); s.concat(nonHeaderHandle, 0, segmentName); baseStorage.create(nonHeaderName); // We reuse this Segment Name since it should have been gone by now. baseStorage.create(segmentName);
@Test @Override public void testFencing() throws Exception { final String segment1 = "segment1"; final String segment2 = "segment2"; @Cleanup val baseStorage = new InMemoryStorage(); @Cleanup val storage = new AsyncStorageWrapper(baseStorage, executorService()); storage.initialize(DEFAULT_EPOCH); // Part 1: Create a segment and verify all operations are allowed. storage.create(segment1, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); SegmentHandle handle1 = storage.openWrite(segment1).join(); verifyAllOperationsSucceed(handle1, storage); // Part 2: Change owner, verify segment operations are not allowed until a call to open() is made. baseStorage.changeOwner(); verifyWriteOperationsFail(handle1, storage); handle1 = storage.openWrite(segment1).join(); verifyAllOperationsSucceed(handle1, storage); // Part 3: Create new segment and verify all operations are allowed. storage.create(segment2, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); SegmentHandle handle2 = storage.openWrite(segment2).join(); verifyAllOperationsSucceed(handle2, storage); // Cleanup. storage.delete(handle1, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); storage.delete(handle2, TIMEOUT).get(TIMEOUT.toMillis(), TimeUnit.MILLISECONDS); }
val baseStorage = new InMemoryStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); while (checkedLength < writtenData.length) { String chunkName = StreamSegmentNameUtils.getSegmentChunkName(SEGMENT_NAME, checkedLength); Assert.assertTrue("Inexistent SegmentChunk: " + chunkName, baseStorage.exists(chunkName)); val chunkInfo = baseStorage.getStreamSegmentInfo(chunkName); int expectedLength = (int) Math.min(DEFAULT_ROLLING_POLICY.getMaxLength(), writtenData.length - checkedLength); Assert.assertEquals("Unexpected SegmentChunk length for: " + chunkName, expectedLength, chunkInfo.getLength());
private void initializeInternal(long epoch) { super.initialize(epoch); }
public void append(SegmentHandle handle, InputStream data, int length) { this.wrappedSyncStorage.append(handle, data, length); }
void resetWriter() { this.writer.close(); this.baseStorage.changeOwner(); this.writer = new StorageWriter(this.config, this.dataSource, this.storage, this.createProcessors, executorService()); }
@Override public void concat(SegmentHandle targetHandle, long offset, String sourceSegment) throws StreamSegmentException { maybeThrow(sourceSegment, this.concatFailure); super.concat(targetHandle, offset, sourceSegment); }
@Override public void delete(SegmentHandle handle) throws StreamSegmentNotExistsException { maybeThrow(handle.getSegmentName(), this.deleteFailure); super.delete(handle); }
@Override protected Storage createStorage() { return wrap(new InMemoryStorage()); } }
@Override protected Storage createStorage() { return new AsyncStorageWrapper(new RollingStorage(new InMemoryStorage(), DEFAULT_ROLLING_POLICY), executorService()); }
private SegmentHandle readHandle(String segmentName) { return InMemoryStorage.newHandle(segmentName, true); }
@Override public void close() { this.index.close(); this.cacheManager.close(); this.cacheFactory.close(); this.storage.close(); this.memoryStorage.close(); }
TestContext(WriterConfig config, WriterFactory.CreateProcessors createProcessors) { this.metadata = new MetadataBuilder(CONTAINER_ID).build(); this.baseStorage = new InMemoryStorage(); this.storage = new TestStorage(this.baseStorage, executorService()); this.storage.initialize(1); this.config = config; this.createProcessors = createProcessors; this.transactionIds = new HashMap<>(); val dataSourceConfig = new TestWriterDataSource.DataSourceConfig(); dataSourceConfig.autoInsertCheckpointFrequency = METADATA_CHECKPOINT_FREQUENCY; this.dataSource = new TestWriterDataSource(this.metadata, executorService(), dataSourceConfig); this.writer = new StorageWriter(this.config, this.dataSource, this.storage, this.createProcessors, executorService()); }
@Override public CompletableFuture<SegmentHandle> openRead(String streamSegmentName) { return CompletableFuture.completedFuture(InMemoryStorage.newHandle(streamSegmentName, true)); }
/** * Tests the ability to auto-refresh a Write Handle upon offset disagreement. */ @Test public void testRefreshHandleBadOffset() throws Exception { // Write small and large writes, alternatively. @Cleanup val baseStorage = new InMemoryStorage(); @Cleanup val s = new RollingStorage(baseStorage, DEFAULT_ROLLING_POLICY); s.initialize(1); s.create(SEGMENT_NAME); val h1 = s.openWrite(SEGMENT_NAME); val h2 = s.openWrite(SEGMENT_NAME); // Open now, before writing, so we force a refresh. byte[] data = "data".getBytes(); s.write(h1, 0, new ByteArrayInputStream(data), data.length); s.write(h2, data.length, new ByteArrayInputStream(data), data.length); // Check that no file has exceeded its maximum length. byte[] expectedData = new byte[data.length * 2]; System.arraycopy(data, 0, expectedData, 0, data.length); System.arraycopy(data, 0, expectedData, data.length, data.length); checkWrittenData(expectedData, h2, s); }