/** * Converts a proto type to a wire type. * * @param tieredPIdentity the proto type to convert * @return the converted wire type */ public static TieredIdentity fromProto(alluxio.grpc.TieredIdentity tieredPIdentity) { return new TieredIdentity(tieredPIdentity.getTiersList().stream().map(GrpcUtils::fromProto) .collect(Collectors.toList())); }
String value = null; if (scriptIdentity != null) { LocalityTier scriptTier = scriptIdentity.getTier(i); Preconditions.checkState(scriptTier.getTierName().equals(tierName)); value = scriptTier.getValue(); tiers.set(0, new LocalityTier(Constants.LOCALITY_NODE, name)); return new TieredIdentity(tiers);
/** * @param tieredIdentity the tiered identity * @param identities the tiered identities to compare to * @param resolveIpAddress whether or not to resolve IP addresses for node locality * @return the identity closest to this one. If none of the identities match, the first identity * is returned */ public static Optional<TieredIdentity> nearest(TieredIdentity tieredIdentity, List<TieredIdentity> identities, boolean resolveIpAddress) { if (identities.isEmpty()) { return Optional.empty(); } for (TieredIdentity.LocalityTier tier : tieredIdentity.getTiers()) { for (TieredIdentity identity : identities) { for (TieredIdentity.LocalityTier otherTier : identity.getTiers()) { if (tier != null && matches(tier, otherTier, resolveIpAddress)) { return Optional.of(identity); } } } } return Optional.of(identities.get(0)); }
/** * @param other a tiered identity to compare to * @return whether the top tier of this tiered identity matches the top tier of other */ public boolean topTiersMatch(TieredIdentity other) { return mTiers.get(0).equals(other.getTier(0)); }
if (nearest.isPresent()) { dataSource = locations.stream().map(BlockLocation::getWorkerAddress) .filter(addr -> addr.getTieredIdentity().equals(nearest.get())).findFirst().get(); if (mTieredIdentity.getTier(0).getTierName().equals(Constants.LOCALITY_NODE) && mTieredIdentity.topTiersMatch(nearest.get())) { dataSourceType = BlockInStreamSource.LOCAL; } else {
public void string() { TieredIdentity identity = new TieredIdentity( Arrays.asList(new LocalityTier("k1", "v1"), new LocalityTier("k2", "v2"))); assertEquals("TieredIdentity(k1=v1, k2=v2)", identity.toString()); }
/** * Creates a new instance of {@link WorkerNetAddress} from thrift representation. * * @param address the thrift net address * @return the instance */ public static WorkerNetAddress fromThrift(alluxio.thrift.WorkerNetAddress address) { TieredIdentity tieredIdentity = TieredIdentity.fromThrift(address.getTieredIdentity()); if (tieredIdentity == null) { // This means the worker is pre-1.7.0. We handle this in post-1.7.0 clients by filling out // the tiered identity using the hostname field. tieredIdentity = new TieredIdentity( Arrays.asList(new LocalityTier(Constants.LOCALITY_NODE, address.getHost()))); } return new WorkerNetAddress() .setDataPort(address.getDataPort()) .setDomainSocketPath(address.getDomainSocketPath()) .setHost(address.getHost()) .setRpcPort(address.getRpcPort()) .setTieredIdentity(tieredIdentity) .setWebPort(address.getWebPort()); }
@Override @Nullable public WorkerNetAddress getWorkerForNextBlock(Iterable<BlockWorkerInfo> workerInfoList, long blockSizeBytes) { List<BlockWorkerInfo> shuffledWorkers = Lists.newArrayList(workerInfoList); Collections.shuffle(shuffledWorkers); // Workers must have enough capacity to hold the block. List<BlockWorkerInfo> candidateWorkers = shuffledWorkers.stream() .filter(worker -> worker.getCapacityBytes() >= blockSizeBytes) .collect(Collectors.toList()); // Try finding a worker based on nearest tiered identity. List<TieredIdentity> identities = candidateWorkers.stream() .map(worker -> worker.getNetAddress().getTieredIdentity()) .filter(Objects::nonNull) .collect(Collectors.toList()); Optional<TieredIdentity> nearest = TieredIdentityUtils.nearest(mTieredIdentity, identities, mCompareNodeIps); if (!nearest.isPresent()) { return null; } // Map back to the worker with the nearest tiered identity. return candidateWorkers.stream() .filter(worker -> worker.getNetAddress().getTieredIdentity().equals(nearest.get())) .map(worker -> worker.getNetAddress()) .findFirst().orElse(null); }
/** * @return a net address of thrift construct */ public alluxio.thrift.WorkerNetAddress toThrift() { alluxio.thrift.WorkerNetAddress address = new alluxio.thrift.WorkerNetAddress(); address.setHost(mHost); address.setRpcPort(mRpcPort); address.setDataPort(mDataPort); address.setWebPort(mWebPort); address.setDomainSocketPath(mDomainSocketPath); if (mTieredIdentity != null) { address.setTieredIdentity(mTieredIdentity.toThrift()); } return address; }
@Test public void chooseClosestTierAvoidEviction() throws Exception { List<BlockWorkerInfo> workers = new ArrayList<>(); workers.add(worker(Constants.GB, Constants.MB, "node2", "rack3")); workers.add(worker(Constants.GB, 0, "node3", "rack2")); workers.add(worker(Constants.GB, 0, "node4", "rack3")); FileWriteLocationPolicy policy; WorkerNetAddress chosen; // local rack with enough availability policy = new LocalFirstAvoidEvictionPolicy( TieredIdentityFactory.fromString("node=node2,rack=rack3", mConf), mConf); chosen = policy.getWorkerForNextBlock(workers, Constants.GB); assertEquals("node4", chosen.getTieredIdentity().getTier(0).getValue()); }
/** * @return the tiered identity */ public TieredIdentity getTieredIdentity() { if (mTieredIdentity != null) { return mTieredIdentity; } return new TieredIdentity(Arrays.asList(new LocalityTier(Constants.LOCALITY_NODE, mHost))); }
String value = null; if (scriptIdentity != null) { LocalityTier scriptTier = scriptIdentity.getTier(i); Preconditions.checkState(scriptTier.getTierName().equals(tierName)); value = scriptTier.getValue(); tiers.set(0, new LocalityTier(Constants.LOCALITY_NODE, name)); return new TieredIdentity(tiers);
@Test public void chooseClosestTier() throws Exception { List<BlockWorkerInfo> workers = new ArrayList<>(); workers.add(worker(Constants.GB, "node2", "rack3")); workers.add(worker(Constants.GB, "node3", "rack2")); workers.add(worker(Constants.GB, "node4", "rack3")); LocalFirstPolicy policy; WorkerNetAddress chosen; // local rack policy = LocalFirstPolicy.create(TieredIdentityFactory.fromString("node=node1,rack=rack2", sConf), sConf.getBoolean(PropertyKey.LOCALITY_COMPARE_NODE_IP)); chosen = policy.getWorkerForNextBlock(workers, Constants.GB); assertEquals("rack2", chosen.getTieredIdentity().getTier(1).getValue()); // local node policy = LocalFirstPolicy.create(TieredIdentityFactory.fromString("node=node4,rack=rack3", sConf), sConf.getBoolean(PropertyKey.LOCALITY_COMPARE_NODE_IP)); chosen = policy.getWorkerForNextBlock(workers, Constants.GB); assertEquals("node4", chosen.getTieredIdentity().getTier(0).getValue()); }
/** * Converts wire type to proto type. * * @param tieredIdentity the wire representation to convert * @return the converted proto representation */ public static alluxio.grpc.TieredIdentity toProto(TieredIdentity tieredIdentity) { return alluxio.grpc.TieredIdentity.newBuilder() .addAllTiers( tieredIdentity.getTiers().stream().map(GrpcUtils::toProto).collect(Collectors.toList())) .build(); }
tieredIdentity.add(new LocalityTier(localityTier, value)); return new TieredIdentity(tieredIdentity);
/** * @param other a tiered identity to compare to * @return whether the top tier of this tiered identity matches the top tier of other */ public boolean topTiersMatch(TieredIdentity other) { return mTiers.get(0).equals(other.getTier(0)); }
public static TieredIdentity createRandomTieredIdentity() { return new TieredIdentity( Arrays.asList(createRandomLocalityTier(), createRandomLocalityTier())); }
private BlockWorkerInfo worker(long capacity, String node, String rack) { WorkerNetAddress address = new WorkerNetAddress(); List<LocalityTier> tiers = new ArrayList<>(); if (node != null && !node.isEmpty()) { address.setHost(node); tiers.add(new LocalityTier(Constants.LOCALITY_NODE, node)); } if (rack != null && !rack.isEmpty()) { tiers.add(new LocalityTier(Constants.LOCALITY_RACK, rack)); } address.setTieredIdentity(new TieredIdentity(tiers)); return new BlockWorkerInfo(address, capacity, 0); } }
private BlockWorkerInfo worker(long capacity, long used, String node, String rack) { WorkerNetAddress address = new WorkerNetAddress(); List<LocalityTier> tiers = new ArrayList<>(); if (node != null && !node.isEmpty()) { address.setHost(node); tiers.add(new LocalityTier(Constants.LOCALITY_NODE, node)); } if (rack != null && !rack.isEmpty()) { tiers.add(new LocalityTier(Constants.LOCALITY_RACK, rack)); } address.setTieredIdentity(new TieredIdentity(tiers)); return new BlockWorkerInfo(address, capacity, used); } }