/** * Creates a new instance of {@link Address} from proto representation. * * @param address the proto representation of a master address * @return the instance */ public static Address fromProto(alluxio.grpc.NetAddress address) { return new Address(address); }
@Override public void getMasterId(GetMasterIdPRequest request, StreamObserver<GetMasterIdPResponse> responseObserver) { NetAddress masterAddress = request.getMasterAddress(); RpcUtils.call(LOG, (RpcUtils.RpcCallableThrowsIOException<GetMasterIdPResponse>) () -> { return GetMasterIdPResponse.newBuilder() .setMasterId(mMetaMaster.getMasterId(Address.fromProto(masterAddress))).build(); }, "getMasterId", "request=%s", responseObserver, request); }
/** * Fills the configuration map. * * @param targetMap the map to fill * @param recordMap the map to get data from */ private void fillConfMap(Map<PropertyKey, Map<Optional<String>, List<String>>> targetMap, Map<Address, List<ConfigRecord>> recordMap) { for (Map.Entry<Address, List<ConfigRecord>> record : recordMap.entrySet()) { Address address = record.getKey(); String addressStr = String.format("%s:%s", address.getHost(), address.getRpcPort()); for (ConfigRecord conf : record.getValue()) { PropertyKey key = conf.getKey(); if (key.getConsistencyLevel() == ConsistencyCheckLevel.IGNORE) { continue; } Optional<String> value = conf.getValue(); targetMap.putIfAbsent(key, new HashMap<>()); Map<Optional<String>, List<String>> values = targetMap.get(key); values.putIfAbsent(value, new ArrayList<>()); values.get(value).add(addressStr); } } } }
/** * Creates a new instance of {@link DefaultMetaMaster}. * * @param blockMaster a block master handle * @param masterContext the context for Alluxio master * @param executorServiceFactory a factory for creating the executor service to use for running * maintenance threads */ DefaultMetaMaster(BlockMaster blockMaster, CoreMasterContext masterContext, ExecutorServiceFactory executorServiceFactory) { super(masterContext, new SystemClock(), executorServiceFactory); mMasterAddress = new Address().setHost(ServerConfiguration.getOrDefault(PropertyKey.MASTER_HOSTNAME, "localhost")) .setRpcPort(mPort); mBlockMaster = blockMaster; mBlockMaster.registerLostWorkerFoundListener(mWorkerConfigStore::lostNodeFound); mBlockMaster.registerWorkerLostListener(mWorkerConfigStore::handleNodeLost); mBlockMaster.registerNewWorkerConfListener(mWorkerConfigStore::registerNewConf); if (URIUtils.isLocalFilesystem(ServerConfiguration .get(PropertyKey.MASTER_MOUNT_TABLE_ROOT_UFS))) { mUfs = UnderFileSystem.Factory .create("/", UnderFileSystemConfiguration.defaults()); } else { mUfs = UnderFileSystem.Factory.createForRoot(ServerConfiguration.global()); } }
/** * Returns a master id for a master address. * * @param address the address to get a master id for * @return a master id */ public long getId(final Address address) throws IOException { return retryRPC(() -> mClient .getMasterId(GetMasterIdPRequest.newBuilder().setMasterAddress(address.toProto()).build()) .getMasterId()); }
@Override public GetMasterIdTResponse getMasterId(final MasterNetAddress address, GetMasterIdTOptions options) throws AlluxioTException { return RpcUtils.call(LOG, (RpcCallable<GetMasterIdTResponse>) () -> new GetMasterIdTResponse( mMetaMaster.getMasterId(Address.fromThrift(address))), "GetMasterId", "address=%s, options=%s", address, options); }
/** * Returns a master id for a master address. * * @param address the address to get a master id for * @return a master id */ public synchronized long getId(final Address address) throws IOException { return retryRPC(() -> mClient .getMasterId(address.toThrift(), new GetMasterIdTOptions()).getMasterId()); }
/** * Creates a new instance of {@link DefaultMetaMaster}. * * @param blockMaster a block master handle * @param masterContext the context for Alluxio master * @param executorServiceFactory a factory for creating the executor service to use for running * maintenance threads */ DefaultMetaMaster(BlockMaster blockMaster, MasterContext masterContext, ExecutorServiceFactory executorServiceFactory) { super(masterContext, new SystemClock(), executorServiceFactory); mSafeModeManager = masterContext.getSafeModeManager(); mStartTimeMs = masterContext.getStartTimeMs(); mMasterAddress = new Address().setHost(Configuration.getOrDefault(PropertyKey.MASTER_HOSTNAME, "localhost")) .setRpcPort(masterContext.getPort()); mBlockMaster = blockMaster; mBlockMaster.registerLostWorkerFoundListener(mWorkerConfigStore::lostNodeFound); mBlockMaster.registerWorkerLostListener(mWorkerConfigStore::handleNodeLost); mBlockMaster.registerNewWorkerConfListener(mWorkerConfigStore::registerNewConf); }
/** * Re-register a lost worker or complete registration after getting a worker id. * * @param workerId the worker id to register */ @Nullable private MasterWorkerInfo registerWorkerInternal(long workerId) { for (IndexedSet<MasterWorkerInfo> workers: Arrays.asList(mTempWorkers, mLostWorkers)) { MasterWorkerInfo worker = workers.getFirstByField(ID_INDEX, workerId); if (worker == null) { continue; } synchronized (worker) { worker.updateLastUpdatedTimeMs(); mWorkers.add(worker); workers.remove(worker); if (workers == mLostWorkers) { for (Consumer<Address> function : mLostWorkerFoundListeners) { function.accept(new Address(worker.getWorkerAddress().getHost(), worker.getWorkerAddress().getRpcPort())); } LOG.warn("A lost worker {} has requested its old id {}.", worker.getWorkerAddress(), worker.getId()); } } return worker; } return null; }
/** * Fills the configuration map. * * @param targetMap the map to fill * @param recordMap the map to get data from */ private void fillConfMap(Map<PropertyKey, Map<Optional<String>, List<String>>> targetMap, Map<Address, List<ConfigRecord>> recordMap) { for (Map.Entry<Address, List<ConfigRecord>> record : recordMap.entrySet()) { Address address = record.getKey(); String addressStr = String.format("%s:%s", address.getHost(), address.getRpcPort()); for (ConfigRecord conf : record.getValue()) { PropertyKey key = conf.getKey(); if (key.getConsistencyLevel() == ConsistencyCheckLevel.IGNORE) { continue; } Optional<String> value = conf.getValue(); targetMap.putIfAbsent(key, new HashMap<>()); Map<Optional<String>, List<String>> values = targetMap.get(key); values.putIfAbsent(value, new ArrayList<>()); values.get(value).add(addressStr); } } } }
@Override public void heartbeat() { long masterWorkerTimeoutMs = ServerConfiguration.getMs(PropertyKey.MASTER_WORKER_TIMEOUT_MS); for (MasterWorkerInfo worker : mWorkers) { synchronized (worker) { final long lastUpdate = mClock.millis() - worker.getLastUpdatedTimeMs(); if (lastUpdate > masterWorkerTimeoutMs) { LOG.error("The worker {}({}) timed out after {}ms without a heartbeat!", worker.getId(), worker.getWorkerAddress(), lastUpdate); mLostWorkers.add(worker); mWorkers.remove(worker); WorkerNetAddress workerAddress = worker.getWorkerAddress(); for (Consumer<Address> function : mWorkerLostListeners) { function.accept(new Address(workerAddress.getHost(), workerAddress.getRpcPort())); } processWorkerRemovedBlocks(worker, worker.getBlocks()); } } } }
for (BiConsumer<Address, List<ConfigProperty>> function : mWorkerRegisteredListeners) { WorkerNetAddress workerAddress = worker.getWorkerAddress(); function.accept(new Address(workerAddress.getHost(), workerAddress.getRpcPort()), options.getConfigsList());
@Test public void registerNewConfUnknownProperty() { Address testAddress = new Address("test", 0); ServerConfigurationStore configStore = new ServerConfigurationStore(); configStore.registerNewConf(testAddress, Arrays.asList(ConfigProperty.newBuilder().setName("unknown.property").build() )); Map<Address, List<ConfigRecord>> confMap = configStore.getConfMap(); assertTrue(confMap.containsKey(testAddress)); assertEquals("unknown.property", confMap.get(testAddress).get(0).getKey().getName()); }
@Before public void before() { PropertyKey keyEnforce = PropertyKey.ZOOKEEPER_ELECTION_PATH; PropertyKey keyWarn = PropertyKey.WORKER_FREE_SPACE_TIMEOUT; mConfigListOne = Arrays.asList( ConfigProperty.newBuilder().setName(keyEnforce.getName()).setSource("Test") .setValue("Value").build(), ConfigProperty.newBuilder().setName(keyWarn.getName()).setSource("Test").setValue("Value2") .build()); mConfigListTwo = Arrays.asList( ConfigProperty.newBuilder().setName(keyEnforce.getName()).setSource("Test") .setValue("Value3").build(), ConfigProperty.newBuilder().setName(keyWarn.getName()).setSource("Test").setValue("Value4") .build()); Random random = new Random(); mAddressOne = new Address(RandomStringUtils.randomAlphanumeric(10), random.nextInt()); mAddressTwo = new Address(RandomStringUtils.randomAlphanumeric(10), random.nextInt()); }
/** * Creates a new instance of {@link Address} from thrift representation. * * @param address the thrift representation of a master address * @return the instance */ public static Address fromThrift(alluxio.thrift.MasterNetAddress address) { return new Address(address); }
/** * Re-register a lost worker or complete registration after getting a worker id. * * @param workerId the worker id to register */ @Nullable private MasterWorkerInfo registerWorkerInternal(long workerId) { for (IndexedSet<MasterWorkerInfo> workers: Arrays.asList(mTempWorkers, mLostWorkers)) { MasterWorkerInfo worker = workers.getFirstByField(ID_INDEX, workerId); if (worker == null) { continue; } synchronized (worker) { worker.updateLastUpdatedTimeMs(); mWorkers.add(worker); workers.remove(worker); if (workers == mLostWorkers) { for (Consumer<Address> function : mLostWorkerFoundListeners) { function.accept(new Address(worker.getWorkerAddress().getHost(), worker.getWorkerAddress().getRpcPort())); } LOG.warn("A lost worker {} has requested its old id {}.", worker.getWorkerAddress(), worker.getId()); } } return worker; } return null; }
for (BiConsumer<Address, List<ConfigProperty>> function : mWorkerRegisteredListeners) { WorkerNetAddress workerAddress = worker.getWorkerAddress(); function.accept(new Address(workerAddress.getHost(), workerAddress.getRpcPort()), wireConfigList);
@Override public void heartbeat() { long masterWorkerTimeoutMs = Configuration.getMs(PropertyKey.MASTER_WORKER_TIMEOUT_MS); for (MasterWorkerInfo worker : mWorkers) { synchronized (worker) { final long lastUpdate = mClock.millis() - worker.getLastUpdatedTimeMs(); if (lastUpdate > masterWorkerTimeoutMs) { LOG.error("The worker {}({}) timed out after {}ms without a heartbeat!", worker.getId(), worker.getWorkerAddress(), lastUpdate); mLostWorkers.add(worker); mWorkers.remove(worker); WorkerNetAddress workerAddress = worker.getWorkerAddress(); for (Consumer<Address> function : mWorkerLostListeners) { function.accept(new Address(workerAddress.getHost(), workerAddress.getRpcPort())); } processWorkerRemovedBlocks(worker, worker.getBlocks()); } } } }