/** * Gets a default set of configuration values, in absence of any real configuration. * These configuration values are the default ones from all component configurations, except that it will * create only one container to host segments. */ public static ServiceBuilderConfig getDefaultConfig() { // All component configs should have defaults built-in, so no need to override them here. return new Builder() .include(ServiceConfig.builder().with(ServiceConfig.CONTAINER_COUNT, 1)) .build(); }
private void checkForExtraLogs(Context context) throws Exception { val maxLogId = context.serviceConfig.getContainerCount() * 10; for (int logId = context.serviceConfig.getContainerCount(); logId < maxLogId; logId++) { @Cleanup DebugLogWrapper log = context.logFactory.createDebugLogWrapper(logId); val m = log.fetchMetadata(); if (m != null) { throw new Exception(String.format("Discovered BookKeeperLog %d which is beyond the maximum log id (%d) as specified in the configuration.", logId, context.serviceConfig.getContainerCount() - 1)); } } }
public void start() throws Exception { Exceptions.checkNotClosed(this.closed, this); log.info("Initializing metrics provider ..."); MetricsProvider.initialize(builderConfig.getConfig(MetricsConfig::builder)); statsProvider = MetricsProvider.getMetricsProvider(); statsProvider.start(); log.info("Initializing ZooKeeper Client ..."); this.zkClient = createZKClient(); log.info("Initializing Service Builder ..."); this.serviceBuilder.initialize(); log.info("Creating StreamSegmentService ..."); StreamSegmentStore service = this.serviceBuilder.createStreamSegmentService(); log.info("Creating TableStoreService ..."); TableStore tableStoreService = this.serviceBuilder.createTableStoreService(); log.info("Creating Segment Stats recorder ..."); segmentStatsFactory = new SegmentStatsFactory(); SegmentStatsRecorder statsRecorder = segmentStatsFactory .createSegmentStatsRecorder(service, builderConfig.getConfig(AutoScalerConfig::builder)); TokenVerifierImpl tokenVerifier = new TokenVerifierImpl(builderConfig.getConfig(AutoScalerConfig::builder)); this.listener = new PravegaConnectionListener(this.serviceConfig.isEnableTls(), this.serviceConfig.getListeningIPAddress(), this.serviceConfig.getListeningPort(), service, tableStoreService, statsRecorder, tokenVerifier, this.serviceConfig.getCertFile(), this.serviceConfig.getKeyFile(), this.serviceConfig.isReplyWithStackTraceOnError()); this.listener.startListening(); log.info("PravegaConnectionListener started successfully."); log.info("StreamSegmentService started."); }
private void attachZKSegmentManager(ServiceBuilder builder) { builder.withContainerManager(setup -> new ZKSegmentContainerManager(setup.getContainerRegistry(), this.zkClient, new Host(this.serviceConfig.getPublishedIPAddress(), this.serviceConfig.getPublishedPort(), null), setup.getCoreExecutor())); }
@Test public void testListeningAndPublicIPAndPort() { ServiceConfig cfg1 = ServiceConfig.builder() .with(ServiceConfig.CONTAINER_COUNT, 1) .with(ServiceConfig.LISTENING_IP_ADDRESS, "myhost") .build(); Assert.assertTrue("Published IP and port should default to listening IP and port", cfg1.getListeningIPAddress().equals(cfg1.getPublishedIPAddress()) && cfg1.getListeningPort() == cfg1.getPublishedPort()); ServiceConfig cfg2 = ServiceConfig.builder() .with(ServiceConfig.CONTAINER_COUNT, 1) .with(ServiceConfig.LISTENING_IP_ADDRESS, "myhost") .build(); Assert.assertTrue("Published IP should default to listening IP even when ports are different", !cfg2.getListeningIPAddress().equals(cfg2.getPublishedIPAddress()) && cfg2.getListeningPort() == cfg2.getPublishedPort()); ServiceConfig cfg3 = ServiceConfig.builder() .with(ServiceConfig.CONTAINER_COUNT, 1) .with(ServiceConfig.LISTENING_IP_ADDRESS, "myhost") .build(); Assert.assertTrue("When specified publishing IP and port should differ from listening IP and port", !cfg3.getListeningIPAddress().equals(cfg3.getPublishedIPAddress()) && cfg3.getListeningPort() != cfg3.getPublishedPort());
/** * Creates a ServiceBuilderConfig based on the given builder, by attaching the correct RocksDB file path based on the instance. * * @param configBuilder The ServiceBuilderConfig.Builder to base from (this builder will not be touched). * @param instanceId The instance id of the Service to build (for least interference, different instances should have * different Ids so that shared resources (i.e., RocksDB) can be setup appropriately). * @return A ServiceBuilderConfig instance. */ protected ServiceBuilderConfig getBuilderConfig(ServiceBuilderConfig.Builder configBuilder, int instanceId) { String id = Integer.toString(instanceId); return configBuilder .makeCopy() .include(ServiceConfig.builder().with(ServiceConfig.INSTANCE_ID, id)) .include(RocksDBConfig.builder().with(RocksDBConfig.DATABASE_DIR, Paths.get(getRocksDBDir().toString(), id).toString())) .build(); }
protected SegmentToContainerMapper createSegmentToContainerMapper(ServiceConfig serviceConfig) { return new SegmentToContainerMapper(serviceConfig.getContainerCount()); }
private static ServiceBuilderConfig getReadOnlyBuilderConfig() { val baseConfig = getBuilderConfig(); val props = new Properties(); baseConfig.forEach(props::put); return ServiceBuilderConfig.builder() .include(props) .include(ServiceConfig.builder() .with(ServiceConfig.READONLY_SEGMENT_STORE, true)) .build(); }
private void collectAllReferencedLedgerIds(Collection<Long> referencedLedgerIds, Context context) throws Exception { referencedLedgerIds.clear(); for (int logId = 0; logId < context.serviceConfig.getContainerCount(); logId++) { @Cleanup DebugLogWrapper log = context.logFactory.createDebugLogWrapper(logId); val m = log.fetchMetadata(); if (m == null) { continue; } for (val lm : m.getLedgers()) { referencedLedgerIds.add(lm.getLedgerId()); } } }
b.include(ServiceConfig.builder() .with(ServiceConfig.CONTAINER_COUNT, testConfig.getContainerCount())); b.include(BookKeeperConfig.builder()
@Override public void execute() throws Exception { ensureArgCount(0); // Loop through all known log ids and fetch their metadata. @Cleanup val context = createContext(); for (int logId = 0; logId < context.serviceConfig.getContainerCount(); logId++) { @Cleanup DebugLogWrapper log = context.logFactory.createDebugLogWrapper(logId); val m = log.fetchMetadata(); outputLogSummary(logId, m); } }
private ServiceBuilder createReadOnlyBuilder(int instanceId) throws Exception { // Copy base config properties to a new object. val props = new Properties(); this.configBuilder.build().forEach(props::put); // Create a new config (so we don't alter the base one) and set the ReadOnlySegmentStore to true). val configBuilder = ServiceBuilderConfig.builder() .include(props) .include(ServiceConfig.builder() .with(ServiceConfig.READONLY_SEGMENT_STORE, true)); val builder = createBuilder(configBuilder, instanceId); builder.initialize(); return builder; }
private static ServiceBuilderConfig getBuilderConfig() { return ServiceBuilderConfig .builder() .include(ServiceConfig.builder() .with(ServiceConfig.CONTAINER_COUNT, 1) .with(ServiceConfig.THREAD_POOL_SIZE, 3) .with(ServiceConfig.LISTENING_PORT, TestUtils.getAvailableListenPort())) .build(); }
.include(System.getProperties()) .include(authProps) .include(ServiceConfig.builder() .with(ServiceConfig.CONTAINER_COUNT, containerCount) .with(ServiceConfig.THREAD_POOL_SIZE, THREADPOOL_SIZE)
/** * Generates a new ServiceBuilderConfig.Builder with hardcoded defaults, in case these are not supplied via other means, * such as a config file or System Properties. */ private static ServiceBuilderConfig.Builder getDefaultServiceBuilderConfig() { return ServiceBuilderConfig .builder() .include(ServiceConfig.builder() .with(ServiceConfig.THREAD_POOL_SIZE, 80) .with(ServiceConfig.CACHE_POLICY_MAX_TIME, 600) .with(ServiceConfig.CACHE_POLICY_MAX_SIZE, 4 * 1024 * 1024 * 1024L) .with(ServiceConfig.CERT_FILE, "../config/cert.pem") .with(ServiceConfig.KEY_FILE, "../config/key.pem")) .include(DurableLogConfig.builder() .with(DurableLogConfig.CHECKPOINT_COMMIT_COUNT, 100) .with(DurableLogConfig.CHECKPOINT_MIN_COMMIT_COUNT, 100) .with(DurableLogConfig.CHECKPOINT_TOTAL_COMMIT_LENGTH, 100 * 1024 * 1024L)) .include(ReadIndexConfig.builder() .with(ReadIndexConfig.MEMORY_READ_MIN_LENGTH, 128 * 1024)) .include(ContainerConfig.builder() .with(ContainerConfig.SEGMENT_METADATA_EXPIRATION_SECONDS, ContainerConfig.MINIMUM_SEGMENT_METADATA_EXPIRATION_SECONDS) .with(ContainerConfig.MAX_ACTIVE_SEGMENT_COUNT, 500)) // This is for those tests that use BookKeeper for Tier1. .include(BookKeeperConfig.builder() .with(BookKeeperConfig.BK_LEDGER_MAX_SIZE, Integer.MAX_VALUE) .with(BookKeeperConfig.ZK_METADATA_PATH, "/pravega/selftest/segmentstore/containers") .with(BookKeeperConfig.BK_LEDGER_PATH, TestConfig.BK_LEDGER_PATH)); }
.builder() .include(ServiceConfig .builder() .with(ServiceConfig.CONTAINER_COUNT, 4) .with(ServiceConfig.THREAD_POOL_SIZE, THREADPOOL_SIZE_SEGMENT_STORE)