/** * Creates a ServiceBuilderConfig based on the given builder, by attaching the correct RocksDB file path based on the instance. * * @param configBuilder The ServiceBuilderConfig.Builder to base from (this builder will not be touched). * @param instanceId The instance id of the Service to build (for least interference, different instances should have * different Ids so that shared resources (i.e., RocksDB) can be setup appropriately). * @return A ServiceBuilderConfig instance. */ protected ServiceBuilderConfig getBuilderConfig(ServiceBuilderConfig.Builder configBuilder, int instanceId) { String id = Integer.toString(instanceId); return configBuilder .makeCopy() .include(ServiceConfig.builder().with(ServiceConfig.INSTANCE_ID, id)) .include(RocksDBConfig.builder().with(RocksDBConfig.DATABASE_DIR, Paths.get(getRocksDBDir().toString(), id).toString())) .build(); }
.include(ServiceConfig .builder() .with(ServiceConfig.CONTAINER_COUNT, 4) .with(ServiceConfig.CACHE_POLICY_MAX_SIZE, 64 * 1024 * 1024L) .with(ServiceConfig.CACHE_POLICY_MAX_TIME, 30)) .include(ContainerConfig .builder() .with(ContainerConfig.SEGMENT_METADATA_EXPIRATION_SECONDS, ContainerConfig.MINIMUM_SEGMENT_METADATA_EXPIRATION_SECONDS)) .include(DurableLogConfig .builder() .with(DurableLogConfig.CHECKPOINT_MIN_COMMIT_COUNT, 10) .with(DurableLogConfig.CHECKPOINT_COMMIT_COUNT, 100) .with(DurableLogConfig.CHECKPOINT_TOTAL_COMMIT_LENGTH, 10 * 1024 * 1024L)) .include(ReadIndexConfig .builder() .with(ReadIndexConfig.MEMORY_READ_MIN_LENGTH, 512) // Need this for truncation testing. .with(ReadIndexConfig.STORAGE_READ_ALIGNMENT, 1024)) .include(WriterConfig .builder() .with(WriterConfig.FLUSH_THRESHOLD_BYTES, 1)
.include(System.getProperties()) .include(authProps) .include(ServiceConfig.builder() .with(ServiceConfig.CONTAINER_COUNT, containerCount) .with(ServiceConfig.THREAD_POOL_SIZE, THREADPOOL_SIZE) ServiceConfig.StorageType.INMEMORY : ServiceConfig.StorageType.FILESYSTEM)) .include(DurableLogConfig.builder() .with(DurableLogConfig.CHECKPOINT_COMMIT_COUNT, 100) .with(DurableLogConfig.CHECKPOINT_MIN_COMMIT_COUNT, 100) .with(DurableLogConfig.CHECKPOINT_TOTAL_COMMIT_LENGTH, 100 * 1024 * 1024L)) .include(AutoScalerConfig.builder() .with(AutoScalerConfig.CONTROLLER_URI, (this.enableTls ? "tls" : "tcp") + "://localhost:" + controllerPorts[0]) .with(AutoScalerConfig.TLS_ENABLED, this.enableTls) .with(AutoScalerConfig.TLS_CERT_FILE, this.certFile)) .include(MetricsConfig.builder() .with(MetricsConfig.ENABLE_STATISTICS, enableMetrics)); nodeServiceStarter[segmentStoreId] = new ServiceStarter(configBuilder.build()); nodeServiceStarter[segmentStoreId].start();
b.include(System.getProperty(TestConfig.CONFIG_FILE_PROPERTY_NAME, TestConfig.DEFAULT_CONFIG_FILE_NAME)); b.include(System.getProperties()); b.include(overrides); val testConfig = b.build().getConfig(TestConfig::builder); int bkWriteQuorum = Math.min(3, testConfig.getBookieCount()); b.include(ServiceConfig.builder() .with(ServiceConfig.CONTAINER_COUNT, testConfig.getContainerCount())); b.include(BookKeeperConfig.builder() .with(BookKeeperConfig.ZK_ADDRESS, TestConfig.LOCALHOST + ":" + testConfig.getZkPort()) .with(BookKeeperConfig.BK_ACK_QUORUM_SIZE, bkWriteQuorum) .with(BookKeeperConfig.BK_ENSEMBLE_SIZE, bkWriteQuorum)); if (testConfig.isMetricsEnabled()) { b.include(MetricsConfig.builder() .with(MetricsConfig.ENABLE_STATISTICS, true) .with(MetricsConfig.ENABLE_CSV_REPORTER, true) return b.build();
this.configBuilder.include(BookKeeperConfig .builder() .with(BookKeeperConfig.ZK_ADDRESS, "localhost:" + zkPort)
@Cleanup val context = createContext(); val readIndexConfig = getCommandArgs().getState().getConfigBuilder().build().getConfig(ReadIndexConfig::builder);
/** * Tests the Builder for ServiceBuilderConfig. */ @Test public void testBuilder() throws Exception { final int propertyCount = 20; // Create three raw Properties. val p1 = buildProperties(0, propertyCount / 2, 0); val p2 = buildProperties(propertyCount / 4, propertyCount / 2, 100); val p3 = buildProperties(propertyCount / 2, propertyCount / 2, 1000); // Save #1 to a temp file. @Cleanup("delete") File tmpFile = File.createTempFile("pravega-", "-servicebuildertest"); try (val writer = new FileWriter(tmpFile)) { p1.store(writer, ""); } // Create a builder and load up #1 from file val b = ServiceBuilderConfig.builder().include(tmpFile.getPath()); val expected = new Properties(); expected.putAll(p1); verifyContents("Unexpected contents after: file. ", expected, b); // Include #2. b.include(p2); expected.putAll(p2); verifyContents("Unexpected contents after: file + Properties.", expected, b); // Include a test config builder with #3. b.include(new ConfigBuilderSupplier<>(p3, s -> s)); expected.putAll(p3); verifyContents("Unexpected contents after: file + Properties + ConfigBuilder.", expected, b); }
/** * Generates a new ServiceBuilderConfig.Builder with hardcoded defaults, in case these are not supplied via other means, * such as a config file or System Properties. */ private static ServiceBuilderConfig.Builder getDefaultServiceBuilderConfig() { return ServiceBuilderConfig .builder() .include(ServiceConfig.builder() .with(ServiceConfig.THREAD_POOL_SIZE, 80) .with(ServiceConfig.CACHE_POLICY_MAX_TIME, 600) .with(ServiceConfig.CACHE_POLICY_MAX_SIZE, 4 * 1024 * 1024 * 1024L) .with(ServiceConfig.CERT_FILE, "../config/cert.pem") .with(ServiceConfig.KEY_FILE, "../config/key.pem")) .include(DurableLogConfig.builder() .with(DurableLogConfig.CHECKPOINT_COMMIT_COUNT, 100) .with(DurableLogConfig.CHECKPOINT_MIN_COMMIT_COUNT, 100) .with(DurableLogConfig.CHECKPOINT_TOTAL_COMMIT_LENGTH, 100 * 1024 * 1024L)) .include(ReadIndexConfig.builder() .with(ReadIndexConfig.MEMORY_READ_MIN_LENGTH, 128 * 1024)) .include(ContainerConfig.builder() .with(ContainerConfig.SEGMENT_METADATA_EXPIRATION_SECONDS, ContainerConfig.MINIMUM_SEGMENT_METADATA_EXPIRATION_SECONDS) .with(ContainerConfig.MAX_ACTIVE_SEGMENT_COUNT, 500)) // This is for those tests that use BookKeeper for Tier1. .include(BookKeeperConfig.builder() .with(BookKeeperConfig.BK_LEDGER_MAX_SIZE, Integer.MAX_VALUE) .with(BookKeeperConfig.ZK_METADATA_PATH, "/pravega/selftest/segmentstore/containers") .with(BookKeeperConfig.BK_LEDGER_PATH, TestConfig.BK_LEDGER_PATH)); }
@Before public void setUp() throws Exception { ServiceBuilderConfig config = ServiceBuilderConfig .builder() .include(System.getProperties()) .build(); SingleNodeConfig conf = config.getConfig(SingleNodeConfig::builder); localPravega = LocalPravegaEmulator.builder() .controllerPort(TestUtils.getAvailableListenPort()) .segmentStorePort(TestUtils.getAvailableListenPort()) .zkPort(TestUtils.getAvailableListenPort()) .restServerPort(TestUtils.getAvailableListenPort()) .enableRestServer(restEnabled) .enableAuth(authEnabled) .enableTls(tlsEnabled) .certFile("../config/cert.pem") .keyFile("../config/key.pem") .jksKeyFile("../config/bookie.keystore.jks") .jksTrustFile("../config/bookie.truststore.jks") .keyPasswordFile("../config/bookie.keystore.jks.passwd") .passwdFile("../config/passwd") .userName("admin") .passwd("1111_aaaa") .build(); localPravega.start(); }
/** * Creates a new Context to be used by the BookKeeper command. * * @return A new Context. * @throws DurableDataLogException If the BookKeeperLogFactory could not be initialized. */ @Override protected Context createContext() throws DurableDataLogException { val serviceConfig = getServiceConfig(); val containerConfig = getCommandArgs().getState().getConfigBuilder().build().getConfig(ContainerConfig::builder); val bkConfig = getCommandArgs().getState().getConfigBuilder() .include(BookKeeperConfig.builder().with(BookKeeperConfig.ZK_ADDRESS, serviceConfig.getZkURL())) .build().getConfig(BookKeeperConfig::builder); val zkClient = createZKClient(); val factory = new BookKeeperLogFactory(bkConfig, zkClient, getCommandArgs().getState().getExecutor()); try { factory.initialize(); } catch (DurableDataLogException ex) { zkClient.close(); throw ex; } val bkAdmin = new BookKeeperAdmin(factory.getBookKeeperClient()); return new Context(serviceConfig, containerConfig, bkConfig, zkClient, factory, bkAdmin); }
/** * Creates a new Context to be used by the BookKeeper command. * * @return A new Context. * @throws DurableDataLogException If the BookKeeperLogFactory could not be initialized. */ protected Context createContext() throws DurableDataLogException { val serviceConfig = getServiceConfig(); val bkConfig = getCommandArgs().getState().getConfigBuilder() .include(BookKeeperConfig.builder().with(BookKeeperConfig.ZK_ADDRESS, serviceConfig.getZkURL())) .build().getConfig(BookKeeperConfig::builder); val zkClient = createZKClient(); val factory = new BookKeeperLogFactory(bkConfig, zkClient, getCommandArgs().getState().getExecutor()); try { factory.initialize(); } catch (DurableDataLogException ex) { zkClient.close(); throw ex; } val bkAdmin = new BookKeeperAdmin(factory.getBookKeeperClient()); return new Context(serviceConfig, bkConfig, zkClient, factory, bkAdmin); }
/** * Starts BookKeeper. */ @Before public void setUp() throws Exception { super.setUp(); endpoint = "http://127.0.0.1:" + TestUtils.getAvailableListenPort(); URI uri = URI.create(endpoint); filesystemS3 = new S3FileSystemImpl(getBaseDir().toString()); this.configBuilder.include(ExtendedS3StorageConfig.builder() .with(ExtendedS3StorageConfig.BUCKET, "kanpravegatest") .with(ExtendedS3StorageConfig.ACCESS_KEY_ID, "x") .with(ExtendedS3StorageConfig.SECRET_KEY, "x") .with(ExtendedS3StorageConfig.URI, endpoint)); }
/** * Creates a new instance of the AdminCommandState class. * * @throws IOException If unable to read specified config properties file (assuming it exists). */ public AdminCommandState() throws IOException { this.configBuilder = ServiceBuilderConfig.builder(); try { this.configBuilder.include(System.getProperty(ServiceBuilderConfig.CONFIG_FILE_PROPERTY_NAME, "config.properties")); } catch (FileNotFoundException ex) { // Nothing to do here. } }
private ServiceBuilder createReadOnlyBuilder(int instanceId) throws Exception { // Copy base config properties to a new object. val props = new Properties(); this.configBuilder.build().forEach(props::put); // Create a new config (so we don't alter the base one) and set the ReadOnlySegmentStore to true). val configBuilder = ServiceBuilderConfig.builder() .include(props) .include(ServiceConfig.builder() .with(ServiceConfig.READONLY_SEGMENT_STORE, true)); val builder = createBuilder(configBuilder, instanceId); builder.initialize(); return builder; }
private void verifyContents(String message, Properties expected, ServiceBuilderConfig.Builder builder) { AtomicReference<Properties> rawProperties = new AtomicReference<>(); builder.build().getConfig(() -> new TestConfigBuilder<>(s -> s, rawProperties::set)); AssertExtensions.assertContainsSameElements( message, expected.entrySet(), rawProperties.get().entrySet(), (e1, e2) -> { int keyCompare = ((String) e1.getKey()).compareTo((String) e2.getKey()); int valueCompare = ((String) e1.getValue()).compareTo((String) e2.getValue()); return keyCompare * valueCompare + keyCompare + valueCompare; }); }
@Override public void execute() { Properties newValues = new Properties(); getCommandArgs().getArgs().forEach(s -> { String[] items = s.split("="); Preconditions.checkArgument(items.length == 2, "Invalid name=value pair: '%s'.", s); Preconditions.checkArgument(!Strings.isNullOrEmpty(items[0]) && !Strings.isNullOrEmpty(items[1]), "Invalid name=value pair: '%s'.", s); newValues.setProperty(items[0], items[1]); }); Preconditions.checkArgument(newValues.size() > 0, "Expecting at least one argument."); getCommandArgs().getState().getConfigBuilder().include(newValues); }
@Override protected ServiceBuilder createBuilder(ServiceBuilderConfig.Builder builderConfig, int instanceId) { return ServiceBuilder.newInMemoryBuilder(builderConfig.build()) .withStorageFactory(setup -> this.storageFactory) .withDataLogFactory(setup -> this.durableDataLogFactory); }