for (String localityTier : conf.getList(PropertyKey.LOCALITY_ORDER, ",")) { String value = tiers.containsKey(localityTier) ? tiers.get(localityTier) : null; tieredIdentity.add(new LocalityTier(localityTier, value));
dataSource = locations.stream().map(BlockLocation::getWorkerAddress) .filter(addr -> addr.getTieredIdentity().equals(nearest.get())).findFirst().get(); if (mTieredIdentity.getTier(0).getTierName().equals(Constants.LOCALITY_NODE) && mTieredIdentity.topTiersMatch(nearest.get())) { dataSourceType = BlockInStreamSource.LOCAL;
String otherTierName = otherTier.getTierName(); if (!tier.getTierName().equals(otherTierName)) { return false; String otherTierValue = otherTier.getValue(); if (tier.getValue() != null && tier.getValue().equals(otherTierValue)) { return true; if (Constants.LOCALITY_NODE.equals(tier.getTierName())) { try { String tierIpAddress = NetworkAddressUtils.resolveIpAddress(tier.getValue()); String otherTierIpAddress = NetworkAddressUtils.resolveIpAddress(otherTierValue); if (tierIpAddress != null && tierIpAddress.equals(otherTierIpAddress)) {
for (String localityTier : Configuration.getList(PropertyKey.LOCALITY_ORDER, ",")) { String value = tiers.containsKey(localityTier) ? tiers.get(localityTier) : null; tieredIdentity.add(new LocalityTier(localityTier, value));
String otherTierName = otherTier.getTierName(); if (!mTierName.equals(otherTierName)) { return false; String otherTierValue = otherTier.getValue(); if (mValue != null && mValue.equals(otherTierValue)) { return true;
if (scriptIdentity != null) { LocalityTier scriptTier = scriptIdentity.getTier(i); Preconditions.checkState(scriptTier.getTierName().equals(tierName)); value = scriptTier.getValue(); value = Configuration.get(Template.LOCALITY_TIER.format(tierName)); tiers.add(new LocalityTier(tierName, value)); if (tiers.size() > 0 && tiers.get(0).getTierName().equals(Constants.LOCALITY_NODE) && tiers.get(0).getValue() == null) { String name = NetworkAddressUtils.getLocalNodeName(); tiers.set(0, new LocalityTier(Constants.LOCALITY_NODE, name));
/** * Creates a new instance of {@link WorkerNetAddress} from thrift representation. * * @param address the thrift net address * @return the instance */ public static WorkerNetAddress fromThrift(alluxio.thrift.WorkerNetAddress address) { TieredIdentity tieredIdentity = TieredIdentity.fromThrift(address.getTieredIdentity()); if (tieredIdentity == null) { // This means the worker is pre-1.7.0. We handle this in post-1.7.0 clients by filling out // the tiered identity using the hostname field. tieredIdentity = new TieredIdentity( Arrays.asList(new LocalityTier(Constants.LOCALITY_NODE, address.getHost()))); } return new WorkerNetAddress() .setDataPort(address.getDataPort()) .setDomainSocketPath(address.getDomainSocketPath()) .setHost(address.getHost()) .setRpcPort(address.getRpcPort()) .setTieredIdentity(tieredIdentity) .setWebPort(address.getWebPort()); }
/** * @param identities the tiered identities to compare to * @return the identity closest to this one. If none of the identities match, the first identity * is returned */ public Optional<TieredIdentity> nearest(List<TieredIdentity> identities) { if (identities.isEmpty()) { return Optional.empty(); } for (LocalityTier tier : mTiers) { for (TieredIdentity identity : identities) { for (LocalityTier otherTier : identity.mTiers) { if (tier != null && tier.matches(otherTier)) { return Optional.of(identity); } } } } return Optional.of(identities.get(0)); }
/** * @return the tiered identity */ public TieredIdentity getTieredIdentity() { if (mTieredIdentity != null) { return mTieredIdentity; } return new TieredIdentity(Arrays.asList(new LocalityTier(Constants.LOCALITY_NODE, mHost))); }
/** * @param localityTier a Thrift locality tier * @return the corresponding wire type locality tier */ public static LocalityTier fromThrift(alluxio.thrift.LocalityTier localityTier) { return new LocalityTier(localityTier.getTierName(), localityTier.getValue()); }
/** * @param other a tiered identity to compare to * @return whether the top tier of this tiered identity matches the top tier of other */ public boolean topTiersMatch(TieredIdentity other) { return mTiers.get(0).equals(other.getTier(0)); }
@Override public String toString() { String tiers = Joiner.on(", ").join(mTiers.stream() .map(tier -> tier.getTierName() + "=" + tier.getValue()) .collect(Collectors.toList())); return String.format("TieredIdentity(%s)", tiers); }
if (scriptIdentity != null) { LocalityTier scriptTier = scriptIdentity.getTier(i); Preconditions.checkState(scriptTier.getTierName().equals(tierName)); value = scriptTier.getValue(); value = conf.get(Template.LOCALITY_TIER.format(tierName)); tiers.add(new LocalityTier(tierName, value)); if (tiers.size() > 0 && tiers.get(0).getTierName().equals(Constants.LOCALITY_NODE) && tiers.get(0).getValue() == null) { String name = NetworkAddressUtils.getLocalNodeName(conf); tiers.set(0, new LocalityTier(Constants.LOCALITY_NODE, name));
@Test public void chooseClosestTier() throws Exception { List<BlockWorkerInfo> workers = new ArrayList<>(); workers.add(worker(Constants.GB, "node2", "rack3")); workers.add(worker(Constants.GB, "node3", "rack2")); workers.add(worker(Constants.GB, "node4", "rack3")); LocalFirstPolicy policy; WorkerNetAddress chosen; // local rack policy = LocalFirstPolicy.create(TieredIdentityFactory.fromString("node=node1,rack=rack2", sConf), sConf.getBoolean(PropertyKey.LOCALITY_COMPARE_NODE_IP)); chosen = policy.getWorkerForNextBlock(workers, Constants.GB); assertEquals("rack2", chosen.getTieredIdentity().getTier(1).getValue()); // local node policy = LocalFirstPolicy.create(TieredIdentityFactory.fromString("node=node4,rack=rack3", sConf), sConf.getBoolean(PropertyKey.LOCALITY_COMPARE_NODE_IP)); chosen = policy.getWorkerForNextBlock(workers, Constants.GB); assertEquals("node4", chosen.getTieredIdentity().getTier(0).getValue()); }
@Test public void fromScriptClasspath() throws Exception { String customScriptName = "my-alluxio-locality.sh"; File dir = mFolder.newFolder("fromScriptClasspath"); Whitebox.invokeMethod(ClassLoader.getSystemClassLoader(), "addURL", dir.toURI().toURL()); File script = new File(dir, customScriptName); setupScript("node=myhost,rack=myrack,custom=mycustom", script); try (Closeable c = new ConfigurationRule(ImmutableMap.of( PropertyKey.LOCALITY_ORDER, "node,rack,custom", PropertyKey.LOCALITY_SCRIPT, customScriptName), mConfiguration).toResource()) { TieredIdentity identity = TieredIdentityFactory.create(mConfiguration); TieredIdentity expected = new TieredIdentity(Arrays.asList( new LocalityTier("node", "myhost"), new LocalityTier("rack", "myrack"), new LocalityTier("custom", "mycustom"))); assertEquals(expected, identity); } script.delete(); }
@Test public void matchByStringEquality() { LocalityTier lt1 = new LocalityTier("node", "NonResolvableHostname-A"); LocalityTier lt2 = new LocalityTier("node", "NonResolvableHostname-A"); LocalityTier lt3 = new LocalityTier("node", "NonResolvableHostname-B"); LocalityTier lt4 = new LocalityTier("rack", "NonResolvableHostname-A"); LocalityTier lt5 = new LocalityTier("rack", "NonResolvableHostname-B"); LocalityTier lt6 = new LocalityTier("rack", "NonResolvableHostname-B"); LocalityTier lt7 = new LocalityTier("rack", ""); LocalityTier lt8 = new LocalityTier("node", "NonResolvableHostname-A"); LocalityTier lt9 = new LocalityTier("node", ""); assertTrue(TieredIdentityUtils.matches(lt1, lt1, true)); assertTrue(TieredIdentityUtils.matches(lt1, lt2, true)); assertFalse(TieredIdentityUtils.matches(lt2, lt3, true)); assertTrue(TieredIdentityUtils.matches(lt5, lt6, true)); assertFalse(TieredIdentityUtils.matches(lt4, lt5, true)); assertFalse(TieredIdentityUtils.matches(lt6, lt7, true)); assertFalse(TieredIdentityUtils.matches(lt8, lt9, true)); }
@Test public void fromScript() throws Exception { String scriptPath = setupScript("node=myhost,rack=myrack,custom=mycustom", mFolder.newFile()); try (Closeable c = new ConfigurationRule(ImmutableMap.of( PropertyKey.LOCALITY_ORDER, "node,rack,custom", PropertyKey.LOCALITY_SCRIPT, scriptPath), mConfiguration).toResource()) { TieredIdentity identity = TieredIdentityFactory.create(mConfiguration); TieredIdentity expected = new TieredIdentity(Arrays.asList( new LocalityTier("node", "myhost"), new LocalityTier("rack", "myrack"), new LocalityTier("custom", "mycustom"))); assertEquals(expected, identity); } }
@Test public void chooseClosestTierAvoidEviction() throws Exception { List<BlockWorkerInfo> workers = new ArrayList<>(); workers.add(worker(Constants.GB, Constants.MB, "node2", "rack3")); workers.add(worker(Constants.GB, 0, "node3", "rack2")); workers.add(worker(Constants.GB, 0, "node4", "rack3")); FileWriteLocationPolicy policy; WorkerNetAddress chosen; // local rack with enough availability policy = new LocalFirstAvoidEvictionPolicy( TieredIdentityFactory.fromString("node=node2,rack=rack3", mConf), mConf); chosen = policy.getWorkerForNextBlock(workers, Constants.GB); assertEquals("node4", chosen.getTieredIdentity().getTier(0).getValue()); }
@Test public void overrideScript() throws Exception { String scriptPath = setupScript("node=myhost,rack=myrack,custom=mycustom", mFolder.newFile()); try (Closeable c = new ConfigurationRule(ImmutableMap.of( Template.LOCALITY_TIER.format("node"), "overridden", PropertyKey.LOCALITY_ORDER, "node,rack,custom", PropertyKey.LOCALITY_SCRIPT, scriptPath), mConfiguration).toResource()) { TieredIdentity identity = TieredIdentityFactory.create(mConfiguration); TieredIdentity expected = new TieredIdentity(Arrays.asList( new LocalityTier("node", "overridden"), new LocalityTier("rack", "myrack"), new LocalityTier("custom", "mycustom"))); assertEquals(expected, identity); } }
private BlockWorkerInfo worker(long capacity, long used, String node, String rack) { WorkerNetAddress address = new WorkerNetAddress(); List<LocalityTier> tiers = new ArrayList<>(); if (node != null && !node.isEmpty()) { address.setHost(node); tiers.add(new LocalityTier(Constants.LOCALITY_NODE, node)); } if (rack != null && !rack.isEmpty()) { tiers.add(new LocalityTier(Constants.LOCALITY_RACK, rack)); } address.setTieredIdentity(new TieredIdentity(tiers)); return new BlockWorkerInfo(address, capacity, used); } }