public static void main(String[] args) throws Exception { @Cleanup ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig()); serviceBuilder.initialize(); StreamSegmentStore store = serviceBuilder.createStreamSegmentService(); TableStore tableStore = serviceBuilder.createTableStoreService(); @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, StartLocalService.SERVICE_PORT, store, tableStore); server.startListening(); @Cleanup MockStreamManager streamManager = new MockStreamManager(SCOPE, "localhost", StartLocalService.SERVICE_PORT); streamManager.createScope(SCOPE); streamManager.createStream(SCOPE, STREAM_NAME, null); Thread.sleep(60000); System.exit(0); } }
private ServiceBuilder createBuilder() throws Exception { val builder = ServiceBuilder.newInMemoryBuilder(this.configBuilder.build()) .withStorageFactory(setup -> this.storageFactory) .withDataLogFactory(setup -> this.durableDataLogFactory); try { builder.initialize(); } catch (Throwable ex) { builder.close(); throw ex; } return builder; }
/** * Creates a new instance of the ServiceBuilder class which is contained in memory. Any data added to this service will * be lost when the object is garbage collected or the process terminates. * * @param builderConfig The ServiceBuilderConfig to use. * @param executorBuilder A Function that, given a thread count and a pool name, creates a ScheduledExecutorService * with the given number of threads that have the given name as prefix. */ @VisibleForTesting public static ServiceBuilder newInMemoryBuilder(ServiceBuilderConfig builderConfig, ExecutorBuilder executorBuilder) { ServiceConfig serviceConfig = builderConfig.getConfig(ServiceConfig::builder); ServiceBuilder builder; if (serviceConfig.isReadOnlySegmentStore()) { // Only components required for ReadOnly SegmentStore. builder = new ReadOnlyServiceBuilder(builderConfig, serviceConfig, executorBuilder); } else { // Components that are required for general SegmentStore. builder = new ServiceBuilder(builderConfig, serviceConfig, executorBuilder) .withCacheFactory(setup -> new InMemoryCacheFactory()); } // Components that are required for all types of SegmentStore. return builder .withDataLogFactory(setup -> new InMemoryDurableDataLogFactory(setup.getCoreExecutor())) .withContainerManager(setup -> new LocalSegmentContainerManager( setup.getContainerRegistry(), setup.getSegmentToContainerMapper())) .withStorageFactory(setup -> new InMemoryStorageFactory(setup.getStorageExecutor())) .withStreamSegmentStore(setup -> new StreamSegmentService(setup.getContainerRegistry(), setup.getSegmentToContainerMapper())); }
@Override protected ServiceBuilder createBuilder(ServiceBuilderConfig.Builder builderConfig, int instanceId) { return ServiceBuilder.newInMemoryBuilder(builderConfig.build()) .withStorageFactory(setup -> this.storageFactory) .withDataLogFactory(setup -> this.durableDataLogFactory); }
private ServiceBuilder createBuilder(int instanceId) throws Exception { val builder = createBuilder(this.configBuilder, instanceId); try { builder.initialize(); } catch (Throwable ex) { builder.close(); throw ex; } return builder; }
@Before public void setup() throws Exception { // 1. Start ZK zkTestServer = new TestingServerStarter().start(); // 2. Start Pravega SSS ServiceBuilder serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig()); serviceBuilder.initialize(); StreamSegmentStore store = serviceBuilder.createStreamSegmentService(); server = new PravegaConnectionListener(false, servicePort, store, mock(TableStore.class)); server.startListening(); }
public void start() throws Exception { Exceptions.checkNotClosed(this.closed, this); log.info("Initializing metrics provider ..."); MetricsProvider.initialize(builderConfig.getConfig(MetricsConfig::builder)); statsProvider = MetricsProvider.getMetricsProvider(); statsProvider.start(); log.info("Initializing ZooKeeper Client ..."); this.zkClient = createZKClient(); log.info("Initializing Service Builder ..."); this.serviceBuilder.initialize(); log.info("Creating StreamSegmentService ..."); StreamSegmentStore service = this.serviceBuilder.createStreamSegmentService(); log.info("Creating TableStoreService ..."); TableStore tableStoreService = this.serviceBuilder.createTableStoreService(); log.info("Creating Segment Stats recorder ..."); segmentStatsFactory = new SegmentStatsFactory(); SegmentStatsRecorder statsRecorder = segmentStatsFactory .createSegmentStatsRecorder(service, builderConfig.getConfig(AutoScalerConfig::builder)); TokenVerifierImpl tokenVerifier = new TokenVerifierImpl(builderConfig.getConfig(AutoScalerConfig::builder)); this.listener = new PravegaConnectionListener(this.serviceConfig.isEnableTls(), this.serviceConfig.getListeningIPAddress(), this.serviceConfig.getListeningPort(), service, tableStoreService, statsRecorder, tokenVerifier, this.serviceConfig.getCertFile(), this.serviceConfig.getKeyFile(), this.serviceConfig.isReplyWithStackTraceOnError()); this.listener.startListening(); log.info("PravegaConnectionListener started successfully."); log.info("StreamSegmentService started."); }
@Override protected ServiceBuilder createBuilder(ServiceBuilderConfig.Builder configBuilder, int instanceId) { ServiceBuilderConfig builderConfig = getBuilderConfig(configBuilder, instanceId); return ServiceBuilder .newInMemoryBuilder(builderConfig) .withCacheFactory(setup -> new RocksDBCacheFactory(builderConfig.getConfig(RocksDBConfig::builder))) .withStorageFactory(setup -> new HDFSStorageFactory(setup.getConfig(HDFSStorageConfig::builder), setup.getStorageExecutor())) .withDataLogFactory(setup -> new BookKeeperLogFactory(setup.getConfig(BookKeeperConfig::builder), getBookkeeper().getZkClient(), setup.getCoreExecutor())); }
@Before public void setup() throws Exception { InternalLoggerFactory.setDefaultFactory(Slf4JLoggerFactory.INSTANCE); this.serviceBuilder = ServiceBuilder.newInMemoryBuilder(ServiceBuilderConfig.getDefaultConfig()); this.serviceBuilder.initialize(); }
@Test public void appendThroughSegmentClient() throws Exception { String endpoint = "localhost"; int port = TestUtils.getAvailableListenPort(); String testString = "Hello world\n"; String scope = "scope"; String stream = "stream"; StreamSegmentStore store = this.serviceBuilder.createStreamSegmentService(); TableStore tableStore = serviceBuilder.createTableStoreService(); @Cleanup PravegaConnectionListener server = new PravegaConnectionListener(false, port, store, tableStore); server.startListening(); @Cleanup ConnectionFactory clientCF = new ConnectionFactoryImpl(ClientConfig.builder().build()); Controller controller = new MockController(endpoint, port, clientCF); controller.createScope(scope); controller.createStream(scope, stream, StreamConfiguration.builder().build()); SegmentOutputStreamFactoryImpl segmentClient = new SegmentOutputStreamFactoryImpl(controller, clientCF); Segment segment = Futures.getAndHandleExceptions(controller.getCurrentSegments(scope, stream), RuntimeException::new).getSegments().iterator().next(); @Cleanup SegmentOutputStream out = segmentClient.createOutputStreamForSegment(segment, segmentSealedCallback, EventWriterConfig.builder().build(), ""); CompletableFuture<Void> ack = new CompletableFuture<>(); out.write(PendingEvent.withHeader(null, ByteBuffer.wrap(testString.getBytes()), ack)); ack.get(5, TimeUnit.SECONDS); }
@Override protected void shutDown() { this.serviceBuilder.close(); stopBookKeeper(); val zk = this.zkClient; if (zk != null) { zk.close(); this.zkClient = null; } StatsProvider sp = this.statsProvider; if (sp != null) { sp.close(); this.statsProvider = null; } Runtime.getRuntime().removeShutdownHook(this.stopBookKeeperProcess); }
/** * Creates a new instance of the SegmentStoreAdapter class. * * @param testConfig The Test Configuration to use. * @param builderConfig The ServiceBuilderConfig to use. * @param testExecutor An Executor to use for test-related async operations. */ SegmentStoreAdapter(TestConfig testConfig, ServiceBuilderConfig builderConfig, ScheduledExecutorService testExecutor) { this.config = Preconditions.checkNotNull(testConfig, "testConfig"); this.builderConfig = Preconditions.checkNotNull(builderConfig, "builderConfig"); this.storage = new AtomicReference<>(); this.storeExecutor = new AtomicReference<>(); this.testExecutor = Preconditions.checkNotNull(testExecutor, "testExecutor"); this.serviceBuilder = attachDataLogFactory(ServiceBuilder .newInMemoryBuilder(builderConfig) .withCacheFactory(setup -> new RocksDBCacheFactory(setup.getConfig(RocksDBConfig::builder))) .withStorageFactory(setup -> { // We use the Segment Store Executor for the real storage. SingletonStorageFactory factory = new SingletonStorageFactory(setup.getStorageExecutor()); this.storage.set(factory.createStorageAdapter()); // A bit hack-ish, but we need to get a hold of the Store Executor, so we can request snapshots for it. this.storeExecutor.set(setup.getCoreExecutor()); return factory; })); this.stopBookKeeperProcess = new Thread(this::stopBookKeeper); Runtime.getRuntime().addShutdownHook(this.stopBookKeeperProcess); }
@Override protected void startUp() throws Exception { if (this.config.isMetricsEnabled()) { MetricsProvider.initialize(this.builderConfig.getConfig(MetricsConfig::builder)); this.statsProvider = MetricsProvider.getMetricsProvider(); this.statsProvider.start(); } if (this.config.getBookieCount() > 0) { this.bookKeeperService = BookKeeperAdapter.startBookKeeperOutOfProcess(this.config, this.logId); } this.serviceBuilder.initialize(); this.streamSegmentStore = this.serviceBuilder.createStreamSegmentService(); }
private static ServiceBuilder newInlineExecutionInMemoryBuilder(ServiceBuilderConfig config) { return ServiceBuilder.newInMemoryBuilder(config, (size, name) -> new InlineExecutor()) .withStreamSegmentStore(setup -> new SynchronousStreamSegmentStore(new StreamSegmentService( setup.getContainerRegistry(), setup.getSegmentToContainerMapper()))); } }
/** * Creates a new Segment Store Instance, with retries. * Normally we have the Controller coordinating which instances are the rightful survivors, however in this case * we need to simulate some of this behavior ourselves, by being insistent. It is possible that previous instances * meddle with the BKLog ZK metadata during the new instance's initialization, causing the new instance to wrongfully * assume it's not the rightful survivor. A quick retry solves this problem, as there is no other kind of information * available to disambiguate this. */ void createNewInstance() { this.newInstanceRetry.run(() -> { int instanceId = getIteration() + 1; log.info("Starting Instance {}.", instanceId); ServiceBuilder b = createBuilder(instanceId); this.builders.add(b); this.activeStore.set(b.createStreamSegmentService()); this.iteration.incrementAndGet(); log.info("Instance {} Started.", instanceId); return null; }); } }
/** * Creates a new instance of the ServiceBuilder class which is contained in memory. Any data added to this service will * be lost when the object is garbage collected or the process terminates. * * @param builderConfig The ServiceBuilderConfig to use. */ public static ServiceBuilder newInMemoryBuilder(ServiceBuilderConfig builderConfig) { return newInMemoryBuilder(builderConfig, ExecutorServiceHelpers::newScheduledThreadPool); }
private ServiceBuilder attachDataLogFactory(ServiceBuilder builder) { if (this.config.getBookieCount() > 0) { // We were instructed to start at least one Bookie. this.zkClient = CuratorFrameworkFactory .builder() .connectString("localhost:" + this.config.getZkPort()) .namespace("pravega") .retryPolicy(new ExponentialBackoffRetry(1000, 5)) .sessionTimeoutMs(5000) .connectionTimeoutMs(5000) .build(); this.zkClient.start(); return builder.withDataLogFactory(setup -> { BookKeeperConfig bkConfig = setup.getConfig(BookKeeperConfig::builder); return new BookKeeperLogFactory(bkConfig, this.zkClient, setup.getCoreExecutor()); }); } else { // No Bookies -> InMemory Tier1. return builder.withDataLogFactory(setup -> new InMemoryDurableDataLogFactory(setup.getCoreExecutor())); } }
private void attachStorage(ServiceBuilder builder) { builder.withStorageFactory(setup -> { StorageLoader loader = new StorageLoader(); return loader.load(setup, this.serviceConfig.getStorageImplementation().toString(), setup.getStorageExecutor()); }); }
private ServiceBuilder createReadOnlyBuilder(int instanceId) throws Exception { // Copy base config properties to a new object. val props = new Properties(); this.configBuilder.build().forEach(props::put); // Create a new config (so we don't alter the base one) and set the ReadOnlySegmentStore to true). val configBuilder = ServiceBuilderConfig.builder() .include(props) .include(ServiceConfig.builder() .with(ServiceConfig.READONLY_SEGMENT_STORE, true)); val builder = createBuilder(configBuilder, instanceId); builder.initialize(); return builder; }
val tableStore = builder.createTableStoreService(); val tableStore = builder.createTableStoreService(); val tableStore = builder.createTableStoreService(); val tableStore = builder.createTableStoreService();