public void init(SploutConfiguration config) { this.config = config; timeoutThread = new TimeoutThread(config.getLong(DNodeProperties.MAX_QUERY_TIME)); timeoutThread.start(); }
public QNodeHandlerContext(SploutConfiguration config, CoordinationStructures coordinationStructures) { this.config = config; this.coordinationStructures = coordinationStructures; this.thriftClientPoolSize = config.getInt(QNodeProperties.DNODE_POOL_SIZE); this.dnodePoolTimeoutMillis = config.getLong(QNodeProperties.QNODE_DNODE_POOL_TAKE_TIMEOUT); this.replicaBalancer = new ReplicaBalancer(this); initMetrics(); }
public Fetcher(SploutConfiguration config) { tempDir = new File(config.getString(FetcherProperties.TEMP_DIR)); accessKey = config.getString(FetcherProperties.S3_ACCESS_KEY, null); secretKey = config.getString(FetcherProperties.S3_SECRET_KEY, null); downloadBufferSize = config.getInt(FetcherProperties.DOWNLOAD_BUFFER); bytesPerSecThrottle = config.getInt(FetcherProperties.BYTES_PER_SEC_THROTTLE); bytesToReportProgress = config.getLong(FetcherProperties.BYTES_TO_REPORT_PROGRESS); String fsName = config.getString(FetcherProperties.HADOOP_FS_NAME); hadoopConf = new Configuration(); if (fsName != null) { hadoopConf.set("fs.default.name", fsName); } log.info("Created " + Fetcher.class + " with tempDir = " + tempDir); if (bytesPerSecThrottle > 0) { log.info("Throttling at: " + bytesPerSecThrottle + " bytes per sec."); } else { log.warn("No throttling. Fetched data will be transferred at full speed. This may affect query servicing."); } }
public void maybeBalance() { // do this only after warming if (!isWarming.get() && config.getBoolean(QNodeProperties.REPLICA_BALANCE_ENABLE)) { // check if we could balance some partitions List<ReplicaBalancer.BalanceAction> balanceActions = getBalanceActions(); // we will only re-balance versions being served // otherwise strange things may happen: to re-balance a version in the middle of its deployment... Map<String, Long> versionsBeingServed = coordinationStructures.getCopyVersionsBeingServed(); for (ReplicaBalancer.BalanceAction action : balanceActions) { if (versionsBeingServed != null && versionsBeingServed.get(action.getTablespace()) != null && versionsBeingServed.get(action.getTablespace()) == action.getVersion()) { // put if absent + TTL coordinationStructures.getDNodeReplicaBalanceActionsSet().putIfAbsent(action, "", config.getLong(QNodeProperties.BALANCE_ACTIONS_TTL), TimeUnit.SECONDS); } } } }
version, context.getConfig().getLong(QNodeProperties.DEPLOY_TIMEOUT, -1), context.getConfig() .getLong(QNodeProperties.DEPLOY_SECONDS_TO_CHECK_ERROR), context.getConfig().getLong(QNodeProperties.DEPLOY_DNODES_SPREAD_METADATA_TIMEOUT, 180), context.getConfig().getBoolean(QNodeProperties.REPLICA_BALANCE_ENABLE)));
long evictionSeconds = config.getLong(DNodeProperties.EH_CACHE_SECONDS); maxResultsPerQuery = config.getInt(DNodeProperties.MAX_RESULTS_PER_QUERY); int maxCachePools = config.getInt(DNodeProperties.EH_CACHE_N_ELEMENTS); absoluteSlowQueryLimit = config.getLong(DNodeProperties.SLOW_QUERY_ABSOLUTE_LIMIT); deployParallelism = config.getInt(DNodeProperties.DEPLOY_PARALLELISM); factory = new ManagerFactory();