/** * Constructs an {@link AlluxioLogServerProcess} instance. * * @param baseLogsDir base directory to store the logs pushed from remote Alluxio servers */ public AlluxioLogServerProcess(String baseLogsDir) { mPort = Configuration.getInt(PropertyKey.LOGSERVER_PORT); // The log server serves the logging requests from Alluxio servers. mMinNumberOfThreads = Configuration.getInt(PropertyKey.LOGSERVER_THREADS_MIN); mMaxNumberOfThreads = Configuration.getInt(PropertyKey.LOGSERVER_THREADS_MAX); mBaseLogsDir = baseLogsDir; mStopped = true; }
@Override public int getWebPort() { return Configuration.getInt(PropertyKey.MASTER_WEB_PORT); }
/** * Constructs an {@link AsyncUserAccessAuditLogWriter} instance. */ public AsyncUserAccessAuditLogWriter() { int queueCapacity = Configuration.getInt(PropertyKey.MASTER_AUDIT_LOGGING_QUEUE_CAPACITY); mAuditLogEntries = new LinkedBlockingQueue<>(queueCapacity); LOG.info("Audit logging queue capacity is {}.", queueCapacity); mStopped = true; }
/** * Creates a new block worker client pool. * * @param workerAddress the worker address */ public BlockWorkerClientPool(WorkerNetAddress workerAddress) { super(ClientContext.getConf().getInt(Constants.USER_BLOCK_WORKER_CLIENT_THREADS)); mWorkerNetAddress = workerAddress; }
/** * Creates a new lineage master client pool. * * @param masterAddress the master address */ public LineageMasterClientPool(InetSocketAddress masterAddress) { super(ClientContext.getConf().getInt(Constants.USER_LINEAGE_MASTER_CLIENT_THREADS)); mMasterAddress = masterAddress; }
/** * The number of items to query in a single listing chunk. * * @return length of each list request */ protected int getListingChunkLength() { return Configuration.getInt(PropertyKey.UNDERFS_LISTING_LENGTH) > getListingChunkLengthMax() ? getListingChunkLengthMax() : Configuration.getInt(PropertyKey.UNDERFS_LISTING_LENGTH); }
/** * Creates a new block master client pool. * * @param masterAddress the master address */ public BlockMasterClientPool(InetSocketAddress masterAddress) { super(ClientContext.getConf().getInt(Constants.USER_BLOCK_MASTER_CLIENT_THREADS)); mMasterAddress = masterAddress; }
/** * Creates a new file stream master client pool. * * @param masterAddress the master address */ public FileSystemMasterClientPool(InetSocketAddress masterAddress) { super(ClientContext.getConf().getInt(Constants.USER_FILE_MASTER_CLIENT_THREADS)); mMasterAddress = masterAddress; }
/** * Constructor for {@link ZkMasterInquireClient}. * * @param connectDetails connect details * @param electionPath the path of the master election */ private ZkMasterInquireClient(ZkMasterConnectDetails connectDetails, String electionPath) { mConnectDetails = connectDetails; mElectionPath = electionPath; LOG.info("Creating new zookeeper client for {}", connectDetails); // Start the client lazily. mClient = CuratorFrameworkFactory.newClient(connectDetails.getZkAddress(), new ExponentialBackoffRetry(Constants.SECOND_MS, 3)); mMaxTry = Configuration.getInt(PropertyKey.ZOOKEEPER_LEADER_INQUIRY_RETRY_COUNT); }
/** * Initializes the client context singleton, bringing all non-Alluxio configuration state in sync * with the current Alluxio configuration. * * This method is useful for updating parts of {@link ClientContext} which depend on * {@link Configuration} when {@link Configuration} is changed, e.g. the master hostname or port. * This method requires that configuration has been initialized. */ public static void init() { String masterHostname = Preconditions.checkNotNull(sConf.get(Constants.MASTER_HOSTNAME)); int masterPort = sConf.getInt(Constants.MASTER_RPC_PORT); sMasterAddress = new InetSocketAddress(masterHostname, masterPort); sClientMetrics = new ClientMetrics(); sExecutorService = Executors.newFixedThreadPool( sConf.getInt(Constants.USER_BLOCK_WORKER_CLIENT_THREADS), ThreadFactoryUtils.build("block-worker-heartbeat-%d", true)); }
public static UfsAbsentPathCache create(MountTable mountTable) { int numThreads = Configuration.getInt(PropertyKey.MASTER_UFS_PATH_CACHE_THREADS); if (numThreads <= 0) { LOG.info("UfsAbsentPathCache is disabled. {}: {}", PropertyKey.MASTER_UFS_PATH_CACHE_THREADS, numThreads); return new NoopUfsAbsentPathCache(); } return new AsyncUfsAbsentPathCache(mountTable, numThreads); } }
/** * Populate the header with information about master. So we can return to * the master from any page. * * @param request the {@link HttpServletRequest} object * @param response the {@link HttpServletResponse} object * @throws ServletException if the target resource throws this exception */ @Override protected void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { String masterHostName = NetworkAddressUtils.getConnectHost(ServiceType.MASTER_RPC); int masterWebPort = Configuration.getInt(PropertyKey.MASTER_WEB_PORT); request.setAttribute("masterHost", masterHostName); request.setAttribute("masterPort", masterWebPort); getServletContext().getRequestDispatcher("/header.jsp").include(request, response); } }
/** * Constructs an {@link ObjectUnderFileSystem}. * * @param uri the {@link AlluxioURI} used to create this ufs * @param ufsConf UFS configuration */ protected ObjectUnderFileSystem(AlluxioURI uri, UnderFileSystemConfiguration ufsConf) { super(uri, ufsConf); int numThreads = Configuration.getInt(PropertyKey.UNDERFS_OBJECT_STORE_SERVICE_THREADS); mExecutorService = ExecutorServiceFactories.fixedThreadPoolExecutorServiceFactory( "alluxio-underfs-object-service-worker", numThreads).create(); }
int webThreadCount = Configuration.getInt(PropertyKey.WEB_THREADS);
/** * Opens the connection to the worker. And start the heartbeat thread. * * @throws IOException if a non-Alluxio exception occurs */ private synchronized void connectOperation() throws IOException { if (!mConnected) { LOG.info("Connecting to {} worker @ {}", (mIsLocal ? "local" : "remote"), mAddress); TProtocol binaryProtocol = new TBinaryProtocol(AuthenticationUtils.getClientTransport(mConfiguration, mAddress)); mProtocol = new TMultiplexedProtocol(binaryProtocol, getServiceName()); mClient = new BlockWorkerClientService.Client(mProtocol); try { mProtocol.getTransport().open(); } catch (TTransportException e) { LOG.error(e.getMessage(), e); return; } mConnected = true; // only start the heartbeat thread if the connection is successful and if there is not // another heartbeat thread running if (mHeartbeat == null || mHeartbeat.isCancelled() || mHeartbeat.isDone()) { final int interval = mConfiguration.getInt(Constants.USER_HEARTBEAT_INTERVAL_MS); mHeartbeat = mExecutorService.submit(new HeartbeatThread(HeartbeatContext.WORKER_CLIENT, mHeartbeatExecutor, interval)); } } }
@Override public InputStream open(String path, OpenOptions options) throws IOException { IOException thrownException = null; RetryPolicy retryPolicy = new ExponentialBackoffRetry( (int) Configuration.getMs(PropertyKey.UNDERFS_OBJECT_STORE_READ_RETRY_BASE_SLEEP_MS), (int) Configuration.getMs(PropertyKey.UNDERFS_OBJECT_STORE_READ_RETRY_MAX_SLEEP_MS), Configuration.getInt(PropertyKey.UNDERFS_OBJECT_STORE_READ_RETRY_MAX_NUM)); while (retryPolicy.attempt()) { try { return openObject(stripPrefixIfPresent(path), options); } catch (IOException e) { LOG.warn("{} attempt to open {} failed with exception : {}", retryPolicy.getAttemptCount(), path, e.getMessage()); thrownException = e; } } throw thrownException; }
/** * Creates a {@link AlluxioSecondaryMaster}. */ AlluxioSecondaryMaster() { try { URI journalLocation = JournalUtils.getJournalLocation(); mJournalSystem = new JournalSystem.Builder().setLocation(journalLocation).build(); mRegistry = new MasterRegistry(); mSafeModeManager = new DefaultSafeModeManager(); mBackupManager = new BackupManager(mRegistry); mStartTimeMs = System.currentTimeMillis(); mPort = Configuration.getInt(PropertyKey.MASTER_RPC_PORT); // Create masters. MasterUtils.createMasters(mRegistry, new MasterContext(mJournalSystem, mSafeModeManager, mBackupManager, mStartTimeMs, mPort)); // Check that journals of each service have been formatted. if (!mJournalSystem.isFormatted()) { throw new RuntimeException( String.format("Journal %s has not been formatted!", journalLocation)); } mLatch = new CountDownLatch(1); } catch (IOException e) { throw new RuntimeException(e); } }
/** * Constructs a new instance using the given {@link Configuration} object. The mapping cannot be * modified after creation. * * @param levelsProperty the property in the conf that specifies how many levels there are * @param template the format for the conf that identifies the alias for each level */ protected StorageTierAssoc(PropertyKey levelsProperty, PropertyKey.Template template) { int levels = Configuration.getInt(levelsProperty); ImmutableBiMap.Builder<String, Integer> builder = new ImmutableBiMap.Builder<>(); for (int i = 0; i < levels; i++) { String alias = Configuration.get(template.format(i)); builder.put(alias, i); } mAliasToOrdinal = builder.build(); }
String workerDataFolder = Configuration.get(PropertyKey.WORKER_DATA_FOLDER); LOG.info("Formatting worker data folder: {}", workerDataFolder); int storageLevels = Configuration.getInt(PropertyKey.WORKER_TIERED_STORE_LEVELS); for (int level = 0; level < storageLevels; level++) { PropertyKey tierLevelDirPath =
/** * Sets up corresponding directories for tests. */ protected void setupTest() throws IOException { UnderFileSystem ufs = UnderFileSystem.Factory.createForRoot(); String underfsAddress = Configuration.get(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS); // Deletes the ufs dir for this test from to avoid permission problems UnderFileSystemUtils.deleteDirIfExists(ufs, underfsAddress); // Creates ufs dir. This must be called before starting UFS with UnderFileSystemCluster.get(). UnderFileSystemUtils.mkdirIfNotExists(ufs, underfsAddress); // Creates storage dirs for worker int numLevel = Configuration.getInt(PropertyKey.WORKER_TIERED_STORE_LEVELS); for (int level = 0; level < numLevel; level++) { PropertyKey tierLevelDirPath = PropertyKey.Template.WORKER_TIERED_STORE_LEVEL_DIRS_PATH.format(level); String[] dirPaths = Configuration.get(tierLevelDirPath).split(","); for (String dirPath : dirPaths) { FileUtils.createDir(dirPath); } } // Formats the journal Format.format(Format.Mode.MASTER); }