/** * @return the {@link AlluxioConfiguration} backing this {@link FileSystemContext} */ public AlluxioConfiguration getConf() { return mClientContext.getConf(); }
/** * This constructor does not create a copy of the configuration. */ protected ClientContext(ClientContext ctx) { mSubject = ctx.getSubject(); mConf = ctx.getConf(); }
/** * Creates a new client base. * * @param context information required to connect to Alluxio * @param address the address */ public AbstractClient(ClientContext context, InetSocketAddress address) { this(context, address, () -> RetryUtils.defaultClientRetry( context.getConf().getDuration(PropertyKey.USER_RPC_RETRY_MAX_DURATION), context.getConf().getDuration(PropertyKey.USER_RPC_RETRY_BASE_SLEEP_MS), context.getConf().getDuration(PropertyKey.USER_RPC_RETRY_MAX_SLEEP_MS))); }
private String getQualifiedMetricName(String metricName) { try { if (SecurityUtils.isAuthenticationEnabled(mContext.getConf()) && LoginUser.get(mContext.getConf()) != null) { return Metric.getMetricNameWithTags(metricName, CommonMetrics.TAG_USER, LoginUser.get(mContext.getConf()).getName()); } else { return metricName; } } catch (IOException e) { return metricName; } }
private void initializeLocalWorker() throws IOException { List<WorkerNetAddress> addresses = getWorkerAddresses(); if (!addresses.isEmpty()) { if (addresses.get(0).getHost().equals(NetworkAddressUtils.getClientHostName(mClientContext .getConf()))) { mLocalWorker = addresses.get(0); } } mLocalWorkerInitialized = true; }
/** * Creates a file system context with a subject. * * @param ctx the parent subject, set to null if not present */ private FileSystemContext(ClientContext ctx) { mClientContext = ctx; mExecutorService = Executors.newFixedThreadPool(1, ThreadFactoryUtils.build("metrics-master-heartbeat-%d", true)); mClosed = new AtomicBoolean(false); mAppId = ctx.getConf().isSet(PropertyKey.USER_APP_ID) ? ctx.getConf().get(PropertyKey.USER_APP_ID) : IdUtils.createFileSystemContextId(); LOG.info("Created filesystem context with id {}. This ID will be used for identifying info " + "from the client, such as metrics. It can be set manually through the {} property", mAppId, PropertyKey.Name.USER_APP_ID); }
/** * @return an instance of {@link MasterClientContext} */ public MasterClientContext build() { if (mMasterInquireClient == null) { mMasterInquireClient = MasterInquireClient.Factory.create(mContext.getConf()); } return new MasterClientContext(mContext, mMasterInquireClient); } }
/** * @param clientContext the {@link alluxio.ClientContext} containing the subject and configuration * @return the {@link alluxio.client.file.FileSystemContext} */ public static FileSystemContext create(ClientContext clientContext) { Preconditions.checkNotNull(clientContext); FileSystemContext ctx = new FileSystemContext(clientContext); ctx.init(MasterInquireClient.Factory.create(clientContext.getConf())); return ctx; }
/** * Builds the configuration, creating an instance of {@link MasterInquireClient} if none is * specified. * * @return a {@link JobMasterClientContext} */ @Override public JobMasterClientContext build() { if (mMasterInquireClient == null) { mMasterInquireClient = MasterInquireClient.Factory.createForJobMaster(mContext.getConf()); } return new JobMasterClientContext(mContext, mMasterInquireClient); } }
/** * @param subject the parent subject, set to null if not present * @param alluxioConf Alluxio configuration * @return a context */ public static FileSystemContext create(Subject subject, AlluxioConfiguration alluxioConf) { FileSystemContext context = new FileSystemContext(subject, alluxioConf); context.init(MasterInquireClient.Factory.create(context.mClientContext.getConf())); return context; }
/** * Creates a new file system master client pool. * * @param context information for connecting to processes in the cluster * @param masterInquireClient a client for determining the master address */ public FileSystemMasterClientPool(ClientContext context, MasterInquireClient masterInquireClient) { super(context.getConf().getInt(PropertyKey.USER_FILE_MASTER_CLIENT_THREADS)); mClientList = new ConcurrentLinkedQueue<>(); mMasterContext = MasterClientContext.newBuilder(context) .setMasterInquireClient(masterInquireClient).build(); }
/** * Creates a new block master client pool. * * @param context the information required for connecting to Alluxio * @param masterInquireClient a client for determining the master address */ public BlockMasterClientPool(ClientContext context, MasterInquireClient masterInquireClient) { super(context.getConf().getInt(PropertyKey.USER_BLOCK_MASTER_CLIENT_THREADS)); mClientList = new ConcurrentLinkedQueue<>(); mMasterContext = MasterClientContext.newBuilder(context).setMasterInquireClient(masterInquireClient).build(); }
/** * This method is called before the connection is connected. Implementations should add any * additional operations before the connection is connected. * loading the cluster defaults */ protected void beforeConnect() throws IOException { // Bootstrap once for clients if (!isConnected()) { if (!mContext.getConf().clusterDefaultsLoaded()) { mContext.updateWithClusterDefaults(mAddress); } } }
/** * Implements get configuration. * * @param ctx Alluxio client configuration * @param args list of arguments * @return 0 on success, 1 on failures */ public static int getConf(ClientContext ctx, String... args) { return getConfImpl( () -> new RetryHandlingMetaMasterConfigClient(MasterClientContext.newBuilder(ctx).build()), ctx.getConf(), args); }
/** * @return if there are any local workers, the returned list will ONLY contain the local workers, * otherwise a list of all remote workers will be returned */ private List<WorkerNetAddress> getWorkerAddresses() throws IOException { List<WorkerInfo> infos; BlockMasterClient blockMasterClient = mBlockMasterClientPool.acquire(); try { infos = blockMasterClient.getWorkerInfoList(); } finally { mBlockMasterClientPool.release(blockMasterClient); } if (infos.isEmpty()) { throw new UnavailableException(ExceptionMessage.NO_WORKER_AVAILABLE.getMessage()); } // Convert the worker infos into net addresses, if there are local addresses, only keep those List<WorkerNetAddress> workerNetAddresses = new ArrayList<>(); List<WorkerNetAddress> localWorkerNetAddresses = new ArrayList<>(); String localHostname = NetworkAddressUtils.getClientHostName(mClientContext.getConf()); for (WorkerInfo info : infos) { WorkerNetAddress netAddress = info.getAddress(); if (netAddress.getHost().equals(localHostname)) { localWorkerNetAddresses.add(netAddress); } workerNetAddresses.add(netAddress); } return localWorkerNetAddresses.isEmpty() ? workerNetAddresses : localWorkerNetAddresses; }
/** * Closes all the resources associated with the context. Make sure all the resources are released * back to this context before calling this close. After closing the context, all the resources * that acquired from this context might fail. Only call this when you are done with using * the {@link FileSystem} associated with this {@link FileSystemContext}. */ public synchronized void close() throws IOException { if (!mClosed.get()) { mFileSystemMasterClientPool.close(); mFileSystemMasterClientPool = null; mBlockMasterClientPool.close(); mBlockMasterClientPool = null; mMasterInquireClient = null; if (mMetricsMasterClient != null) { ThreadUtils.shutdownAndAwaitTermination(mExecutorService, mClientContext.getConf().getMs(PropertyKey.METRICS_CONTEXT_SHUTDOWN_TIMEOUT)); mMetricsMasterClient.close(); mMetricsMasterClient = null; mClientMasterSync = null; } mLocalWorkerInitialized = false; mLocalWorker = null; mClosed.set(true); } else { LOG.warn("Attempted to close FileSystemContext with app ID {} which has already been closed" + " or not initialized.", mAppId); } }
mClosed.set(false); if (mClientContext.getConf().getBoolean(PropertyKey.USER_METRICS_COLLECTION_ENABLED)) { mExecutorService .submit(new HeartbeatThread(HeartbeatContext.MASTER_METRICS_SYNC, mClientMasterSync, (int) mClientContext.getConf().getMs(PropertyKey.USER_METRICS_HEARTBEAT_INTERVAL_MS), mClientContext.getConf()));
/** * The method the worker should periodically execute to heartbeat back to the master. * * @param metrics a list of client metrics */ public void heartbeat(final List<Metric> metrics) throws IOException { connect(); try { MetricsHeartbeatPRequest.Builder request = MetricsHeartbeatPRequest.newBuilder(); request.setClientId(IdUtils.createOrGetAppIdFromConfig(mContext.getConf())); request.setHostname(NetworkAddressUtils.getClientHostName(mContext.getConf())); request.setOptions(MetricsHeartbeatPOptions.newBuilder().addAllMetrics(metrics).build()); mClient.metricsHeartbeat(request.build()); } catch (io.grpc.StatusRuntimeException e) { throw new UnavailableException(e); } } }
RuntimeConstants.VERSION, getServiceName(), mAddress); mChannel = GrpcChannelBuilder .newBuilder(mAddress, mContext.getConf()) .setSubject(mContext.getSubject()) .build();