public static List<String> getBestLocations(Configuration conf, HDFSBlocksDistribution blockDistribution) { // 3 nodes will contain highly local blocks. So default to 3. return getBestLocations(conf, blockDistribution, 3); }
/** * Compute block locations for snapshot files (which will get the locations for referred hfiles) * only when localityEnabled is true. */ private static List<String> calculateLocationsForInputSplit(Configuration conf, TableDescriptor htd, HRegionInfo hri, Path tableDir, boolean localityEnabled) throws IOException { if (localityEnabled) { // care block locality return getBestLocations(conf, HRegion.computeHDFSBlocksDistribution(conf, htd, hri, tableDir)); } else { // do not care block locality return null; } }
TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); blockDistribution.addHostsAndBlockWeight(new String[] {"h4"}, 1); Assert.assertEquals(Lists.newArrayList("h1"), TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution));
public static List<String> getBestLocations(Configuration conf, HDFSBlocksDistribution blockDistribution) { // 3 nodes will contain highly local blocks. So default to 3. return getBestLocations(conf, blockDistribution, 3); }
public static List<String> getBestLocations(Configuration conf, HDFSBlocksDistribution blockDistribution) { // 3 nodes will contain highly local blocks. So default to 3. return getBestLocations(conf, blockDistribution, 3); }
/** * Compute block locations for snapshot files (which will get the locations for referred hfiles) * only when localityEnabled is true. */ private static List<String> calculateLocationsForInputSplit(Configuration conf, TableDescriptor htd, HRegionInfo hri, Path tableDir, boolean localityEnabled) throws IOException { if (localityEnabled) { // care block locality return getBestLocations(conf, HRegion.computeHDFSBlocksDistribution(conf, htd, hri, tableDir)); } else { // do not care block locality return null; } }
/** * Compute block locations for snapshot files (which will get the locations for referred hfiles) * only when localityEnabled is true. */ private static List<String> calculateLocationsForInputSplit(Configuration conf, TableDescriptor htd, HRegionInfo hri, Path tableDir, boolean localityEnabled) throws IOException { if (localityEnabled) { // care block locality return getBestLocations(conf, HRegion.computeHDFSBlocksDistribution(conf, htd, hri, tableDir)); } else { // do not care block locality return null; } }
public static List<InputSplit> getSplits(Scan scan, SnapshotManifest manifest, List<HRegionInfo> regionManifests, Path restoreDir, Configuration conf) throws IOException { // load table descriptor HTableDescriptor htd = manifest.getTableDescriptor(); Path tableDir = FSUtils.getTableDir(restoreDir, htd.getTableName()); List<InputSplit> splits = new ArrayList<InputSplit>(); for (HRegionInfo hri : regionManifests) { // load region descriptor if (CellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(), hri.getStartKey(), hri.getEndKey())) { // compute HDFS locations from snapshot files (which will get the locations for // referred hfiles) List<String> hosts = getBestLocations(conf, HRegion.computeHDFSBlocksDistribution(conf, htd, hri, tableDir)); int len = Math.min(3, hosts.size()); hosts = hosts.subList(0, len); splits.add(new InputSplit(htd, hri, hosts, scan, restoreDir)); } } return splits; }
TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); blockDistribution.addHostsAndBlockWeight(new String[] {"h4"}, 1); Assert.assertEquals(Lists.newArrayList("h1"), TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution));
TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); blockDistribution.addHostsAndBlockWeight(new String[] {"h4"}, 1); Assert.assertEquals(Lists.newArrayList("h1"), TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution)); TableSnapshotInputFormatImpl.getBestLocations(conf, blockDistribution));