private void setVolumeMeta(List<Fragment> splits, final List<BlockLocation> blockLocations) throws IOException { int locationSize = blockLocations.size(); int splitSize = splits.size(); if (locationSize == 0 || splitSize == 0) return; if (locationSize != splitSize) { // splits and locations don't match up LOG.warn("Number of block locations not equal to number of splits: " + "#locations=" + locationSize + " #splits=" + splitSize); return; } DistributedFileSystem fs = (DistributedFileSystem) this.fs; int lsLimit = conf.getInt(DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT); int blockLocationIdx = 0; Iterator<Fragment> iter = splits.iterator(); while (locationSize > blockLocationIdx) { int subSize = Math.min(locationSize - blockLocationIdx, lsLimit); List<BlockLocation> locations = blockLocations.subList(blockLocationIdx, blockLocationIdx + subSize); //BlockStorageLocation containing additional volume location information for each replica of each block. BlockStorageLocation[] blockStorageLocations = fs.getFileBlockStorageLocations(locations); for (BlockStorageLocation blockStorageLocation : blockStorageLocations) { ((FileFragment)iter.next()).setDiskIds(getDiskIds(blockStorageLocation.getVolumeIds())); blockLocationIdx++; } } LOG.info("# of splits with volumeId " + splitSize); }
private void setVolumeMeta(List<Fragment> splits, final List<BlockLocation> blockLocations) throws IOException { int locationSize = blockLocations.size(); int splitSize = splits.size(); if (locationSize == 0 || splitSize == 0) return; if (locationSize != splitSize) { // splits and locations don't match up LOG.warn("Number of block locations not equal to number of splits: " + "#locations=" + locationSize + " #splits=" + splitSize); return; } DistributedFileSystem fs = (DistributedFileSystem) this.fs; int lsLimit = conf.getInt(DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT); int blockLocationIdx = 0; Iterator<Fragment> iter = splits.iterator(); while (locationSize > blockLocationIdx) { int subSize = Math.min(locationSize - blockLocationIdx, lsLimit); List<BlockLocation> locations = blockLocations.subList(blockLocationIdx, blockLocationIdx + subSize); //BlockStorageLocation containing additional volume location information for each replica of each block. BlockStorageLocation[] blockStorageLocations = fs.getFileBlockStorageLocations(locations); for (BlockStorageLocation blockStorageLocation : blockStorageLocations) { ((FileFragment)iter.next()).setDiskIds(getDiskIds(blockStorageLocation.getVolumeIds())); blockLocationIdx++; } } LOG.info("# of splits with volumeId " + splitSize); }
private void setVolumeMeta(List<FileFragment> splits, final List<BlockLocation> blockLocations) throws IOException { int locationSize = blockLocations.size(); int splitSize = splits.size(); if (locationSize == 0 || splitSize == 0) return; if (locationSize != splitSize) { // splits and locations don't match up LOG.warn("Number of block locations not equal to number of splits: " + "#locations=" + locationSize + " #splits=" + splitSize); return; } DistributedFileSystem fs = (DistributedFileSystem)DistributedFileSystem.get(conf); int lsLimit = conf.getInt(DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT); int blockLocationIdx = 0; Iterator<FileFragment> iter = splits.iterator(); while (locationSize > blockLocationIdx) { int subSize = Math.min(locationSize - blockLocationIdx, lsLimit); List<BlockLocation> locations = blockLocations.subList(blockLocationIdx, blockLocationIdx + subSize); //BlockStorageLocation containing additional volume location information for each replica of each block. BlockStorageLocation[] blockStorageLocations = fs.getFileBlockStorageLocations(locations); for (BlockStorageLocation blockStorageLocation : blockStorageLocations) { iter.next().setDiskIds(getDiskIds(blockStorageLocation.getVolumeIds())); blockLocationIdx++; } } LOG.info("# of splits with volumeId " + splitSize); }