/** * Creates a new instance of the SegmentStoreAdapter class. * * @param testConfig The Test Configuration to use. * @param builderConfig The ServiceBuilderConfig to use. * @param testExecutor An Executor to use for test-related async operations. */ SegmentStoreAdapter(TestConfig testConfig, ServiceBuilderConfig builderConfig, ScheduledExecutorService testExecutor) { this.config = Preconditions.checkNotNull(testConfig, "testConfig"); this.builderConfig = Preconditions.checkNotNull(builderConfig, "builderConfig"); this.storage = new AtomicReference<>(); this.storeExecutor = new AtomicReference<>(); this.testExecutor = Preconditions.checkNotNull(testExecutor, "testExecutor"); this.serviceBuilder = attachDataLogFactory(ServiceBuilder .newInMemoryBuilder(builderConfig) .withCacheFactory(setup -> new RocksDBCacheFactory(setup.getConfig(RocksDBConfig::builder))) .withStorageFactory(setup -> { // We use the Segment Store Executor for the real storage. SingletonStorageFactory factory = new SingletonStorageFactory(setup.getStorageExecutor()); this.storage.set(factory.createStorageAdapter()); // A bit hack-ish, but we need to get a hold of the Store Executor, so we can request snapshots for it. this.storeExecutor.set(setup.getCoreExecutor()); return factory; })); this.stopBookKeeperProcess = new Thread(this::stopBookKeeper); Runtime.getRuntime().addShutdownHook(this.stopBookKeeperProcess); }
private <T> T getSingleton(AtomicReference<T> instance, Function<ComponentSetup, T> creator) { if (instance.get() == null) { instance.set(creator.apply(new ComponentSetup(this))); } return instance.get(); }
/** * Creates a new instance of the ServiceBuilder class which is contained in memory. Any data added to this service will * be lost when the object is garbage collected or the process terminates. * * @param builderConfig The ServiceBuilderConfig to use. * @param executorBuilder A Function that, given a thread count and a pool name, creates a ScheduledExecutorService * with the given number of threads that have the given name as prefix. */ @VisibleForTesting public static ServiceBuilder newInMemoryBuilder(ServiceBuilderConfig builderConfig, ExecutorBuilder executorBuilder) { ServiceConfig serviceConfig = builderConfig.getConfig(ServiceConfig::builder); ServiceBuilder builder; if (serviceConfig.isReadOnlySegmentStore()) { // Only components required for ReadOnly SegmentStore. builder = new ReadOnlyServiceBuilder(builderConfig, serviceConfig, executorBuilder); } else { // Components that are required for general SegmentStore. builder = new ServiceBuilder(builderConfig, serviceConfig, executorBuilder) .withCacheFactory(setup -> new InMemoryCacheFactory()); } // Components that are required for all types of SegmentStore. return builder .withDataLogFactory(setup -> new InMemoryDurableDataLogFactory(setup.getCoreExecutor())) .withContainerManager(setup -> new LocalSegmentContainerManager( setup.getContainerRegistry(), setup.getSegmentToContainerMapper())) .withStorageFactory(setup -> new InMemoryStorageFactory(setup.getStorageExecutor())) .withStreamSegmentStore(setup -> new StreamSegmentService(setup.getContainerRegistry(), setup.getSegmentToContainerMapper())); }
private ServiceBuilder attachDataLogFactory(ServiceBuilder builder) { if (this.config.getBookieCount() > 0) { // We were instructed to start at least one Bookie. this.zkClient = CuratorFrameworkFactory .builder() .connectString("localhost:" + this.config.getZkPort()) .namespace("pravega") .retryPolicy(new ExponentialBackoffRetry(1000, 5)) .sessionTimeoutMs(5000) .connectionTimeoutMs(5000) .build(); this.zkClient.start(); return builder.withDataLogFactory(setup -> { BookKeeperConfig bkConfig = setup.getConfig(BookKeeperConfig::builder); return new BookKeeperLogFactory(bkConfig, this.zkClient, setup.getCoreExecutor()); }); } else { // No Bookies -> InMemory Tier1. return builder.withDataLogFactory(setup -> new InMemoryDurableDataLogFactory(setup.getCoreExecutor())); } }
private void attachRocksDB(ServiceBuilder builder) { builder.withCacheFactory(setup -> new RocksDBCacheFactory(setup.getConfig(RocksDBConfig::builder))); }
private void attachDataLogFactory(ServiceBuilder builder) { builder.withDataLogFactory(setup -> { switch (this.serviceConfig.getDataLogTypeImplementation()) { case BOOKKEEPER: return new BookKeeperLogFactory(setup.getConfig(BookKeeperConfig::builder), this.zkClient, setup.getCoreExecutor()); case INMEMORY: return new InMemoryDurableDataLogFactory(setup.getCoreExecutor()); default: throw new IllegalStateException("Unsupported storage implementation: " + this.serviceConfig.getDataLogTypeImplementation()); } }); }
private static ServiceBuilder newInlineExecutionInMemoryBuilder(ServiceBuilderConfig config) { return ServiceBuilder.newInMemoryBuilder(config, (size, name) -> new InlineExecutor()) .withStreamSegmentStore(setup -> new SynchronousStreamSegmentStore(new StreamSegmentService( setup.getContainerRegistry(), setup.getSegmentToContainerMapper()))); } }
private ReadOnlyServiceBuilder(ServiceBuilderConfig serviceBuilderConfig, ServiceConfig serviceConfig, ExecutorBuilder executorBuilder) { super(serviceBuilderConfig, serviceConfig, executorBuilder); // We attach a LocalSegmentContainerManager, since we only have one Container Running. // Note that withContainerManager() is disabled in ReadOnlyServiceBuilder, hence we must invoke the one on // the parent class. super.withContainerManager(setup -> new LocalSegmentContainerManager(setup.getContainerRegistry(), setup.getSegmentToContainerMapper())); }
@Override protected ServiceBuilder createBuilder(ServiceBuilderConfig.Builder configBuilder, int instanceId) { ServiceBuilderConfig builderConfig = getBuilderConfig(configBuilder, instanceId); return ServiceBuilder .newInMemoryBuilder(builderConfig) .withCacheFactory(setup -> new RocksDBCacheFactory(builderConfig.getConfig(RocksDBConfig::builder))) .withStorageFactory(setup -> new LocalExtendedS3StorageFactory(setup.getConfig(ExtendedS3StorageConfig::builder), setup.getStorageExecutor())) .withDataLogFactory(setup -> new BookKeeperLogFactory(setup.getConfig(BookKeeperConfig::builder), getBookkeeper().getZkClient(), setup.getCoreExecutor())); }
@Override protected ServiceBuilder createBuilder(ServiceBuilderConfig.Builder configBuilder, int instanceId) { ServiceBuilderConfig builderConfig = getBuilderConfig(configBuilder, instanceId); return ServiceBuilder .newInMemoryBuilder(builderConfig) .withCacheFactory(setup -> new RocksDBCacheFactory(builderConfig.getConfig(RocksDBConfig::builder))) .withStorageFactory(setup -> new FileSystemStorageFactory(setup.getConfig(FileSystemStorageConfig::builder), setup.getStorageExecutor())) .withDataLogFactory(setup -> new BookKeeperLogFactory(setup.getConfig(BookKeeperConfig::builder), getBookkeeper().getZkClient(), setup.getCoreExecutor())); } }
private void attachStorage(ServiceBuilder builder) { builder.withStorageFactory(setup -> { StorageLoader loader = new StorageLoader(); return loader.load(setup, this.serviceConfig.getStorageImplementation().toString(), setup.getStorageExecutor()); }); }
private void attachZKSegmentManager(ServiceBuilder builder) { builder.withContainerManager(setup -> new ZKSegmentContainerManager(setup.getContainerRegistry(), this.zkClient, new Host(this.serviceConfig.getPublishedIPAddress(), this.serviceConfig.getPublishedPort(), null), setup.getCoreExecutor())); }
@Override protected ServiceBuilder createBuilder(ServiceBuilderConfig.Builder builderConfig, int instanceId) { return super.createBuilder(builderConfig, instanceId) .withStreamSegmentStore(setup -> { StreamSegmentStore base = new StreamSegmentService(setup.getContainerRegistry(), setup.getSegmentToContainerMapper()); return new SynchronousStreamSegmentStore(base); }); } }
/** * Creates a new instance of TableStore using the components generated by this class. */ public TableStore createTableStoreService() { return getSingleton(this.tableStoreService, setup -> new TableService(setup.getContainerRegistry(), setup.getSegmentToContainerMapper())); }
@Override protected ServiceBuilder createBuilder(ServiceBuilderConfig.Builder configBuilder, int instanceId) { ServiceBuilderConfig builderConfig = getBuilderConfig(configBuilder, instanceId); return ServiceBuilder .newInMemoryBuilder(builderConfig) .withCacheFactory(setup -> new RocksDBCacheFactory(builderConfig.getConfig(RocksDBConfig::builder))) .withStorageFactory(setup -> new HDFSStorageFactory(setup.getConfig(HDFSStorageConfig::builder), setup.getStorageExecutor())) .withDataLogFactory(setup -> new BookKeeperLogFactory(setup.getConfig(BookKeeperConfig::builder), getBookkeeper().getZkClient(), setup.getCoreExecutor())); }