private static List<HostAddress> getHostAddresses(BlockLocation blockLocation) { String[] hosts; try { hosts = blockLocation.getHosts(); } catch (IOException e) { throw new UncheckedIOException(e); } return Arrays.stream(hosts) .map(HostAddress::fromString) .collect(toImmutableList()); }
@Override public String[] getHosts() throws IOException { /** * Unfortunately, the Hadoop API is not precise about if the list returned by BlockLocation.getHosts() contains * the hostnames with their respective domain suffix or not (FQDN or not). We have witnessed both versions, * depending on the cluster's network configuration. As a workaround, we therefore strip every hostname to make * sure it does not contain the domain suffix. */ if (this.hostnames == null) { final String[] hadoopHostnames = blockLocation.getHosts(); this.hostnames = new String[hadoopHostnames.length]; for (int i = 0; i < hadoopHostnames.length; ++i) { this.hostnames[i] = stripHostname(hadoopHostnames[i]); } } return this.hostnames; }
BlockLocation[] locs = fs.getFileBlockLocations(stat, 0, length); for (int i = 0; i < locs.length; i++) { String[] hosts = locs[i].getHosts(); for (int j = 0; j < hosts.length; j++) { locations.add(hosts[j]);
BlockLocation[] locs = fs.getFileBlockLocations(stat, 0, length); for (int i = 0; i < locs.length; i++) { String[] hosts = locs[i].getHosts(); for (int j = 0; j < hosts.length; j++) { locations.add(hosts[j]);
/** * Update blocksDistribution with blockLocations * @param blocksDistribution the hdfs blocks distribution * @param blockLocations an array containing block location */ static public void addToHDFSBlocksDistribution( HDFSBlocksDistribution blocksDistribution, BlockLocation[] blockLocations) throws IOException { for (BlockLocation bl : blockLocations) { String[] hosts = bl.getHosts(); long len = bl.getLength(); blocksDistribution.addHostsAndBlockWeight(hosts, len); } }
int currentRep = locations[i].getHosts().length; if (currentRep != newRep) { if (!printedWarning && currentRep > newRep) {
@Override @Nullable public List<String> getFileLocations(String path, FileLocationOptions options) throws IOException { // If the user has hinted the underlying storage nodes are not co-located with Alluxio // workers, short circuit without querying the locations. if (Boolean.valueOf(mUfsConf.get(PropertyKey.UNDERFS_HDFS_REMOTE))) { return null; } FileSystem hdfs = getFs(); List<String> ret = new ArrayList<>(); try { // The only usage of fileStatus is to get the path in getFileBlockLocations. // In HDFS 2, there is an API getFileBlockLocation(Path path, long offset, long len), // but in HDFS 1, the only API is // getFileBlockLocation(FileStatus stat, long offset, long len). // By constructing the file status manually, we can save one RPC call to getFileStatus. FileStatus fileStatus = new FileStatus(0L, false, 0, 0L, 0L, 0L, null, null, null, new Path(path)); BlockLocation[] bLocations = hdfs.getFileBlockLocations(fileStatus, options.getOffset(), 1); if (bLocations.length > 0) { String[] names = bLocations[0].getHosts(); Collections.addAll(ret, names); } } catch (IOException e) { LOG.warn("Unable to get file location for {} : {}", path, e.getMessage()); } return ret; }
for (String host: blk.getHosts()) { AtomicInteger count = blockCountMap.get(host); if (count == null) {
/** * Convert IGFS affinity block location into Hadoop affinity block location. * * @param block IGFS affinity block location. * @return Hadoop affinity block location. */ private IgfsBlockLocation convertBlockLocation(BlockLocation block) { try { String[] names = block.getNames(); String[] hosts = block.getHosts(); return new IgfsBlockLocationImpl( block.getOffset(), block.getLength(), Arrays.asList(names), Arrays.asList(hosts)); } catch (IOException e) { throw handleSecondaryFsError(e, "Failed convert block location: " + block); } }
@Override public List<OrcSplit> getSplits() throws IOException { List<OrcSplit> splits = Lists.newArrayList(); for (HdfsFileStatusWithId file : fileStatuses) { FileStatus fileStatus = file.getFileStatus(); if (fileStatus.getLen() != 0) { Object fileKey = file.getFileId(); if (fileKey == null && allowSyntheticFileIds) { fileKey = new SyntheticFileId(fileStatus); } TreeMap<Long, BlockLocation> blockOffsets = SHIMS.getLocationsWithOffset(fs, fileStatus); for (Map.Entry<Long, BlockLocation> entry : blockOffsets.entrySet()) { OrcSplit orcSplit = new OrcSplit(fileStatus.getPath(), fileKey, entry.getKey(), entry.getValue().getLength(), entry.getValue().getHosts(), null, isOriginal, true, deltas, -1, fileStatus.getLen()); splits.add(orcSplit); } } } // add uncovered ACID delta splits splits.addAll(super.getSplits()); return splits; }
if (offset + length <= start.getOffset() + start.getLength()) { hosts = start.getHosts(); } else { Map.Entry<Long, BlockLocation> endEntry = locations.floorEntry(offset + length); block.getLength()); if (overlap > 0) { for(String host: block.getHosts()) { LongWritable val = sizes.get(host); if (val == null) { for(String host: block.getHosts()) { if (sizes.containsKey(host)) { if (sizes.get(host).get() >= threshold) {
if (offset + length <= start.getOffset() + start.getLength()) { hosts = start.getHosts(); } else { Map.Entry<Long, BlockLocation> endEntry = locations.floorEntry(offset + length); block.getLength()); if (overlap > 0) { for(String host: block.getHosts()) { LongWritable val = sizes.get(host); if (val == null) { for(String host: block.getHosts()) { if (sizes.containsKey(host)) { if (sizes.get(host).get() >= threshold) {
entry.getValue().getLength(), entry.getValue().getHosts(), null, isOriginal, true, deltas, -1, logicalLen, dir, offsetAndBucket); splits.add(orcSplit);
/** * Compute HDFS blocks distribution of a given file, or a portion of the file * @param fs file system * @param status file status of the file * @param start start position of the portion * @param length length of the portion * @return The HDFS blocks distribution */ static public HDFSBlocksDistribution computeHDFSBlocksDistribution( final FileSystem fs, FileStatus status, long start, long length) throws IOException { HDFSBlocksDistribution blocksDistribution = new HDFSBlocksDistribution(); BlockLocation [] blockLocations = fs.getFileBlockLocations(status, start, length); for(BlockLocation bl : blockLocations) { String [] hosts = bl.getHosts(); long len = bl.getLength(); blocksDistribution.addHostsAndBlockWeight(hosts, len); } return blocksDistribution; }
Set<String> hostsSet = new HashSet<>(); for (BlockLocation location : locations) { hostsSet.addAll(Lists.newArrayList(location.getHosts()));
Collections.sort(expectedWorkerNames, (x, y) -> x.toString().compareTo(y.toString())); String[] actualNames = locations[0].getNames(); String[] actualHosts = locations[0].getHosts(); Arrays.sort(actualNames); Arrays.sort(actualHosts);
final String lookup = lbs[0].getHosts()[0]; StringBuilder sb = new StringBuilder(); for (DataNode dn : cluster.getDataNodes()) {
BlockLocation bl = bls[0]; LOG.info(bl.getHosts().length + " replicas for block 0 in " + logFile + " "); for (int i = 0; i < bl.getHosts().length - 1; i++) { LOG.info(bl.getHosts()[i] + " " + logFile); Assert.assertNotSame(bl.getHosts()[i], host4); String last = bl.getHosts()[bl.getHosts().length - 1]; LOG.info(last + " " + logFile); if (host4.equals(last)) { nbTest++; LOG.info(logFile + " is on the new datanode and is ok"); if (bl.getHosts().length == 3) {
private void addBlocks(VolumeManager fs, String host, ArrayList<String> files, Map<String,Long> totalBlocks, Map<String,Long> localBlocks) throws Exception { long allBlocks = 0; long matchingBlocks = 0; if (!totalBlocks.containsKey(host)) { totalBlocks.put(host, 0L); localBlocks.put(host, 0L); } for (String file : files) { Path filePath = new Path(file); FileSystem ns = fs.getVolumeByPath(filePath).getFileSystem(); FileStatus fileStatus = ns.getFileStatus(filePath); BlockLocation[] fileBlockLocations = ns.getFileBlockLocations(fileStatus, 0, fileStatus.getLen()); for (BlockLocation blockLocation : fileBlockLocations) { allBlocks++; for (String location : blockLocation.getHosts()) { HostAndPort hap = HostAndPort.fromParts(location, 0); if (hap.getHost().equals(host)) { matchingBlocks++; break; } } } } totalBlocks.put(host, allBlocks + totalBlocks.get(host)); localBlocks.put(host, matchingBlocks + localBlocks.get(host)); }
private static Map<String, Object> toJsonMap( final BlockLocation blockLocation) throws IOException { if (blockLocation == null) { return null; } final Map<String, Object> m = new HashMap<>(); m.put("length", blockLocation.getLength()); m.put("offset", blockLocation.getOffset()); m.put("corrupt", blockLocation.isCorrupt()); m.put("storageTypes", toJsonArray(blockLocation.getStorageTypes())); m.put("cachedHosts", blockLocation.getCachedHosts()); m.put("hosts", blockLocation.getHosts()); m.put("names", blockLocation.getNames()); m.put("topologyPaths", blockLocation.getTopologyPaths()); return m; }