/** * Configures RateLimiter based on repository and global settings * * @param repositorySettings repository settings * @param setting setting to use to configure rate limiter * @param defaultRate default limiting rate * @return rate limiter or null of no throttling is needed */ private RateLimiter getRateLimiter(Settings repositorySettings, String setting, ByteSizeValue defaultRate) { ByteSizeValue maxSnapshotBytesPerSec = repositorySettings.getAsBytesSize(setting, settings.getAsBytesSize(setting, defaultRate)); if (maxSnapshotBytesPerSec.getBytes() <= 0) { return null; } else { return new RateLimiter.SimpleRateLimiter(maxSnapshotBytesPerSec.getMbFrac()); } }
public FsBlobStore(Settings settings, Path path) throws IOException { this.path = path; this.readOnly = settings.getAsBoolean("readonly", false); if (!this.readOnly) { Files.createDirectories(path); } this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.fs.buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).getBytes(); }
public HdfsBlobStore(Settings settings, FileSystem fileSystem, Executor executor, Path path) throws IOException { this.fileSystem = fileSystem; this.path = path; if (!fileSystem.exists(path)) { fileSystem.mkdirs(path); } this.bufferSizeInBytes = (int) settings.getAsBytesSize("buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).bytes(); this.executor = executor; }
public FsBlobStore(Settings settings, Path path) throws IOException { super(settings); this.path = path; Files.createDirectories(path); this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.fs.buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).bytes(); }
public FsBlobStore(Settings settings, Path path) throws IOException { super(settings); this.path = path; Files.createDirectories(path); this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.fs.buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).getBytes(); }
/** * Configures RateLimiter based on repository and global settings * * @param repositorySettings repository settings * @param setting setting to use to configure rate limiter * @param defaultRate default limiting rate * @return rate limiter or null of no throttling is needed */ private RateLimiter getRateLimiter(Settings repositorySettings, String setting, ByteSizeValue defaultRate) { ByteSizeValue maxSnapshotBytesPerSec = repositorySettings.getAsBytesSize(setting, settings.getAsBytesSize(setting, defaultRate)); if (maxSnapshotBytesPerSec.getBytes() <= 0) { return null; } else { return new RateLimiter.SimpleRateLimiter(maxSnapshotBytesPerSec.getMbFrac()); } }
/** * Configures RateLimiter based on repository and global settings * * @param repositorySettings repository settings * @param setting setting to use to configure rate limiter * @param defaultRate default limiting rate * @return rate limiter or null of no throttling is needed */ private RateLimiter getRateLimiter(Settings repositorySettings, String setting, ByteSizeValue defaultRate) { ByteSizeValue maxSnapshotBytesPerSec = repositorySettings.getAsBytesSize(setting, settings.getAsBytesSize(setting, defaultRate)); if (maxSnapshotBytesPerSec.getBytes() <= 0) { return null; } else { return new RateLimiter.SimpleRateLimiter(maxSnapshotBytesPerSec.getMbFrac()); } }
/** * Configures RateLimiter based on repository and global settings * * @param repositorySettings repository settings * @param setting setting to use to configure rate limiter * @param defaultRate default limiting rate * @return rate limiter or null of no throttling is needed */ private RateLimiter getRateLimiter(Settings repositorySettings, String setting, ByteSizeValue defaultRate) { ByteSizeValue maxSnapshotBytesPerSec = repositorySettings.getAsBytesSize(setting, settings.getAsBytesSize(setting, defaultRate)); if (maxSnapshotBytesPerSec.getBytes() <= 0) { return null; } else { return new RateLimiter.SimpleRateLimiter(maxSnapshotBytesPerSec.getMbFrac()); } }
public FsBlobStore(Settings settings, Path path) throws IOException { super(settings); this.path = path; this.readOnly = settings.getAsBoolean("readonly", false); if (!this.readOnly) { Files.createDirectories(path); } this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.fs.buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).getBytes(); }
public FsBlobStore(Settings settings, Path path) throws IOException { super(settings); this.path = path; this.readOnly = settings.getAsBoolean("readonly", false); if (!this.readOnly) { Files.createDirectories(path); } this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.fs.buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).getBytes(); }
/** * Configures RateLimiter based on repository and global settings * * @param repositorySettings repository settings * @param setting setting to use to configure rate limiter * @param defaultRate default limiting rate * @return rate limiter or null of no throttling is needed */ private RateLimiter getRateLimiter(RepositorySettings repositorySettings, String setting, ByteSizeValue defaultRate) { ByteSizeValue maxSnapshotBytesPerSec = repositorySettings.settings().getAsBytesSize(setting, settings.getAsBytesSize(setting, defaultRate)); if (maxSnapshotBytesPerSec.bytes() <= 0) { return null; } else { return new RateLimiter.SimpleRateLimiter(maxSnapshotBytesPerSec.mbFrac()); } }
/** * Constructs new read-only URL-based blob store * <p> * The following settings are supported * <dl> * <dt>buffer_size</dt> * <dd>- size of the read buffer, defaults to 100KB</dd> * </dl> * * @param settings settings * @param path base URL */ public URLBlobStore(Settings settings, URL path) { super(settings); this.path = path; this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.uri.buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).getBytes(); }
/** * Constructs new read-only URL-based blob store * <p> * The following settings are supported * <dl> * <dt>buffer_size</dt> * <dd>- size of the read buffer, defaults to 100KB</dd> * </dl> * * @param settings settings * @param path base URL */ public URLBlobStore(Settings settings, URL path) { super(settings); this.path = path; this.bufferSizeInBytes = (int) settings.getAsBytesSize("repositories.uri.buffer_size", new ByteSizeValue(100, ByteSizeUnit.KB)).bytes(); }
@Override public void onRefreshSettings(Settings settings) { String rateLimitingType = settings.get(INDEX_STORE_THROTTLE_TYPE, IndexStore.this.rateLimitingType); if (!rateLimitingType.equals(IndexStore.this.rateLimitingType)) { logger.info("updating index.store.throttle.type from [{}] to [{}]", IndexStore.this.rateLimitingType, rateLimitingType); if (rateLimitingType.equalsIgnoreCase("node")) { IndexStore.this.rateLimitingType = rateLimitingType; IndexStore.this.nodeRateLimiting = true; } else { StoreRateLimiting.Type.fromString(rateLimitingType); IndexStore.this.rateLimitingType = rateLimitingType; IndexStore.this.nodeRateLimiting = false; IndexStore.this.rateLimiting.setType(rateLimitingType); } } ByteSizeValue rateLimitingThrottle = settings.getAsBytesSize(INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, IndexStore.this.rateLimitingThrottle); if (!rateLimitingThrottle.equals(IndexStore.this.rateLimitingThrottle)) { logger.info("updating index.store.throttle.max_bytes_per_sec from [{}] to [{}], note, type is [{}]", IndexStore.this.rateLimitingThrottle, rateLimitingThrottle, IndexStore.this.rateLimitingType); IndexStore.this.rateLimitingThrottle = rateLimitingThrottle; IndexStore.this.rateLimiting.setMaxRate(rateLimitingThrottle); } } }
@Override public void onRefreshSettings(Settings settings) { String rateLimitingType = settings.get(INDICES_STORE_THROTTLE_TYPE, IndicesStore.this.rateLimitingType); // try and parse the type StoreRateLimiting.Type.fromString(rateLimitingType); if (!rateLimitingType.equals(IndicesStore.this.rateLimitingType)) { logger.info("updating indices.store.throttle.type from [{}] to [{}]", IndicesStore.this.rateLimitingType, rateLimitingType); IndicesStore.this.rateLimitingType = rateLimitingType; IndicesStore.this.rateLimiting.setType(rateLimitingType); } ByteSizeValue rateLimitingThrottle = settings.getAsBytesSize(INDICES_STORE_THROTTLE_MAX_BYTES_PER_SEC, IndicesStore.this.rateLimitingThrottle); if (!rateLimitingThrottle.equals(IndicesStore.this.rateLimitingThrottle)) { logger.info("updating indices.store.throttle.max_bytes_per_sec from [{}] to [{}], note, type is [{}]", IndicesStore.this.rateLimitingThrottle, rateLimitingThrottle, IndicesStore.this.rateLimitingType); IndicesStore.this.rateLimitingThrottle = rateLimitingThrottle; IndicesStore.this.rateLimiting.setMaxRate(rateLimitingThrottle); } } }
@Inject public IndexStore(Index index, IndexSettingsService settingsService, IndicesStore indicesStore) { super(index, settingsService.getSettings()); Settings indexSettings = settingsService.getSettings(); this.indicesStore = indicesStore; this.rateLimitingType = indexSettings.get(INDEX_STORE_THROTTLE_TYPE, "none"); if (rateLimitingType.equalsIgnoreCase("node")) { nodeRateLimiting = true; } else { nodeRateLimiting = false; rateLimiting.setType(rateLimitingType); } this.rateLimitingThrottle = indexSettings.getAsBytesSize(INDEX_STORE_THROTTLE_MAX_BYTES_PER_SEC, new ByteSizeValue(0)); rateLimiting.setMaxRate(rateLimitingThrottle); logger.debug("using index.store.throttle.type [{}], with index.store.throttle.max_bytes_per_sec [{}]", rateLimitingType, rateLimitingThrottle); this.settingsService = settingsService; this.settingsService.addListener(applySettings); }
public MergePolicyConfig(ESLogger logger, Settings indexSettings) { this.logger = logger; this.noCFSRatio = parseNoCFSRatio(indexSettings.get(INDEX_COMPOUND_FORMAT, Double.toString(TieredMergePolicy.DEFAULT_NO_CFS_RATIO))); double forceMergeDeletesPctAllowed = indexSettings.getAsDouble("index.merge.policy.expunge_deletes_allowed", DEFAULT_EXPUNGE_DELETES_ALLOWED); // percentage ByteSizeValue floorSegment = indexSettings.getAsBytesSize("index.merge.policy.floor_segment", DEFAULT_FLOOR_SEGMENT); int maxMergeAtOnce = indexSettings.getAsInt("index.merge.policy.max_merge_at_once", DEFAULT_MAX_MERGE_AT_ONCE); int maxMergeAtOnceExplicit = indexSettings.getAsInt("index.merge.policy.max_merge_at_once_explicit", DEFAULT_MAX_MERGE_AT_ONCE_EXPLICIT); // TODO is this really a good default number for max_merge_segment, what happens for large indices, won't they end up with many segments? ByteSizeValue maxMergedSegment = indexSettings.getAsBytesSize("index.merge.policy.max_merged_segment", DEFAULT_MAX_MERGED_SEGMENT); double segmentsPerTier = indexSettings.getAsDouble("index.merge.policy.segments_per_tier", DEFAULT_SEGMENTS_PER_TIER); double reclaimDeletesWeight = indexSettings.getAsDouble("index.merge.policy.reclaim_deletes_weight", DEFAULT_RECLAIM_DELETES_WEIGHT); this.mergesEnabled = indexSettings.getAsBoolean(INDEX_MERGE_ENABLED, true); if (mergesEnabled == false) { logger.warn("[{}] is set to false, this should only be used in tests and can cause serious problems in production environments", INDEX_MERGE_ENABLED); } maxMergeAtOnce = adjustMaxMergeAtOnceIfNeeded(maxMergeAtOnce, segmentsPerTier); mergePolicy.setNoCFSRatio(noCFSRatio); mergePolicy.setForceMergeDeletesPctAllowed(forceMergeDeletesPctAllowed); mergePolicy.setFloorSegmentMB(floorSegment.mbFrac()); mergePolicy.setMaxMergeAtOnce(maxMergeAtOnce); mergePolicy.setMaxMergeAtOnceExplicit(maxMergeAtOnceExplicit); mergePolicy.setMaxMergedSegmentMB(maxMergedSegment.mbFrac()); mergePolicy.setSegmentsPerTier(segmentsPerTier); mergePolicy.setReclaimDeletesWeight(reclaimDeletesWeight); logger.debug("using [tiered] merge mergePolicy with expunge_deletes_allowed[{}], floor_segment[{}], max_merge_at_once[{}], max_merge_at_once_explicit[{}], max_merged_segment[{}], segments_per_tier[{}], reclaim_deletes_weight[{}]", forceMergeDeletesPctAllowed, floorSegment, maxMergeAtOnce, maxMergeAtOnceExplicit, maxMergedSegment, segmentsPerTier, reclaimDeletesWeight); }
public Gatherer start() throws ElasticSearchException { //this.logger = Loggers.getLogger(getClass(), settings.globalSettings(), riverName); this.bulkActions = settings.getAsInt("bulk_actions", 1000); this.bulkSize = settings.getAsBytesSize("bulk_size", new ByteSizeValue(5, ByteSizeUnit.MB)); this.flushInterval = settings.getAsTime("flush_interval", TimeValue.timeValueSeconds(5)); this.concurrentRequests = settings.getAsInt("concurrent_requests", 4); bulkProcessor = BulkProcessor.builder(client, new BulkListener()) .setBulkActions(bulkActions) .setBulkSize(bulkSize) .setFlushInterval(flushInterval) .setConcurrentRequests(concurrentRequests) .build(); return this; }
@Inject public TranslogService(ShardId shardId, IndexSettingsService indexSettingsService, ThreadPool threadPool, IndexShard indexShard) { super(shardId, indexSettingsService.getSettings()); this.threadPool = threadPool; this.indexSettingsService = indexSettingsService; this.indexShard = indexShard; this.flushThresholdOperations = indexSettings.getAsInt(INDEX_TRANSLOG_FLUSH_THRESHOLD_OPS, indexSettings.getAsInt("index.translog.flush_threshold", Integer.MAX_VALUE)); this.flushThresholdSize = indexSettings.getAsBytesSize(INDEX_TRANSLOG_FLUSH_THRESHOLD_SIZE, new ByteSizeValue(512, ByteSizeUnit.MB)); this.flushThresholdPeriod = indexSettings.getAsTime(INDEX_TRANSLOG_FLUSH_THRESHOLD_PERIOD, TimeValue.timeValueMinutes(30)); this.interval = indexSettings.getAsTime(INDEX_TRANSLOG_FLUSH_INTERVAL, timeValueMillis(5000)); this.disableFlush = indexSettings.getAsBoolean(INDEX_TRANSLOG_DISABLE_FLUSH, false); logger.debug("interval [{}], flush_threshold_ops [{}], flush_threshold_size [{}], flush_threshold_period [{}]", interval, flushThresholdOperations, flushThresholdSize, flushThresholdPeriod); this.future = threadPool.schedule(interval, ThreadPool.Names.SAME, new TranslogBasedFlush()); indexSettingsService.addListener(applySettings); }
@Inject public IndicesStore(Settings settings, NodeSettingsService nodeSettingsService, IndicesService indicesService, ClusterService clusterService, TransportService transportService) { super(settings); this.nodeSettingsService = nodeSettingsService; this.indicesService = indicesService; this.clusterService = clusterService; this.transportService = transportService; transportService.registerRequestHandler(ACTION_SHARD_EXISTS, ShardActiveRequest.class, ThreadPool.Names.SAME, new ShardActiveRequestHandler()); // we don't limit by default (we default to CMS's auto throttle instead): this.rateLimitingType = settings.get("indices.store.throttle.type", StoreRateLimiting.Type.NONE.name()); rateLimiting.setType(rateLimitingType); this.rateLimitingThrottle = settings.getAsBytesSize("indices.store.throttle.max_bytes_per_sec", new ByteSizeValue(10240, ByteSizeUnit.MB)); rateLimiting.setMaxRate(rateLimitingThrottle); this.deleteShardTimeout = settings.getAsTime(INDICES_STORE_DELETE_SHARD_TIMEOUT, new TimeValue(30, TimeUnit.SECONDS)); logger.debug("using indices.store.throttle.type [{}], with index.store.throttle.max_bytes_per_sec [{}]", rateLimitingType, rateLimitingThrottle); nodeSettingsService.addListener(applySettings); if (DiscoveryNode.dataNode(settings)) { clusterService.addLast(this); } }