@Override public List<InputSplit> getSplits(JobContext job) throws IOException, InterruptedException { List<InputSplit> results = new ArrayList<>(); for (TableSnapshotInputFormatImpl.InputSplit split : TableSnapshotInputFormatImpl.getSplits(job.getConfiguration())) { results.add(new TableSnapshotRegionSplit(split)); } return results; }
@Override public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { List<TableSnapshotInputFormatImpl.InputSplit> splits = TableSnapshotInputFormatImpl.getSplits(job); InputSplit[] results = new InputSplit[splits.size()]; for (int i = 0; i < splits.size(); i++) { results[i] = new TableSnapshotRegionSplit(splits.get(i)); } return results; }
public static List<InputSplit> getSplits(Configuration conf) throws IOException { String snapshotName = getSnapshotName(conf); Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = rootDir.getFileSystem(conf); SnapshotManifest manifest = getSnapshotManifest(conf, snapshotName, rootDir, fs); List<HRegionInfo> regionInfos = getRegionInfosFromManifest(manifest); // TODO: mapred does not support scan as input API. Work around for now. Scan scan = extractScanFromConf(conf); // the temp dir where the snapshot is restored Path restoreDir = new Path(conf.get(RESTORE_DIR_KEY)); RegionSplitter.SplitAlgorithm splitAlgo = getSplitAlgo(conf); int numSplits = conf.getInt(NUM_SPLITS_PER_REGION, 1); return getSplits(scan, manifest, regionInfos, restoreDir, conf, splitAlgo, numSplits); }
TableSnapshotInputFormatImpl.getSplits(scan, manifest, regionInfos, restoreDir, conf); rtn.addAll(splits);
public static List<InputSplit> getSplits(Scan scan, SnapshotManifest manifest, List<HRegionInfo> regionManifests, Path restoreDir, Configuration conf, RegionSplitter.SplitAlgorithm sa, int numSplits) throws IOException { // load table descriptor TableDescriptor htd = manifest.getTableDescriptor(); Path tableDir = FSUtils.getTableDir(restoreDir, htd.getTableName()); boolean localityEnabled = conf.getBoolean(SNAPSHOT_INPUTFORMAT_LOCALITY_ENABLED_KEY, SNAPSHOT_INPUTFORMAT_LOCALITY_ENABLED_DEFAULT); List<InputSplit> splits = new ArrayList<>(); for (HRegionInfo hri : regionManifests) { // load region descriptor if (numSplits > 1) { byte[][] sp = sa.split(hri.getStartKey(), hri.getEndKey(), numSplits, true); for (int i = 0; i < sp.length - 1; i++) { if (PrivateCellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(), sp[i], sp[i + 1])) { List<String> hosts = calculateLocationsForInputSplit(conf, htd, hri, tableDir, localityEnabled); Scan boundedScan = new Scan(scan); if (scan.getStartRow().length == 0) { boundedScan.withStartRow(sp[i]); } else { boundedScan.withStartRow( Bytes.compareTo(scan.getStartRow(), sp[i]) > 0 ? scan.getStartRow() : sp[i]); }
@Override public List<InputSplit> getSplits(JobContext job) throws IOException, InterruptedException { List<InputSplit> results = new ArrayList<>(); for (TableSnapshotInputFormatImpl.InputSplit split : TableSnapshotInputFormatImpl.getSplits(job.getConfiguration())) { results.add(new TableSnapshotRegionSplit(split)); } return results; }
@Override public List<InputSplit> getSplits(JobContext job) throws IOException, InterruptedException { List<InputSplit> results = new ArrayList<>(); for (TableSnapshotInputFormatImpl.InputSplit split : TableSnapshotInputFormatImpl.getSplits(job.getConfiguration())) { results.add(new TableSnapshotRegionSplit(split)); } return results; }
@Override public List<InputSplit> getSplits(JobContext job) throws IOException, InterruptedException { List<InputSplit> results = new ArrayList<InputSplit>(); for (TableSnapshotInputFormatImpl.InputSplit split : TableSnapshotInputFormatImpl.getSplits(job.getConfiguration())) { results.add(new TableSnapshotRegionSplit(split)); } return results; }
@Override public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { List<TableSnapshotInputFormatImpl.InputSplit> splits = TableSnapshotInputFormatImpl.getSplits(job); InputSplit[] results = new InputSplit[splits.size()]; for (int i = 0; i < splits.size(); i++) { results[i] = new TableSnapshotRegionSplit(splits.get(i)); } return results; }
@Override public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { List<TableSnapshotInputFormatImpl.InputSplit> splits = TableSnapshotInputFormatImpl.getSplits(job); InputSplit[] results = new InputSplit[splits.size()]; for (int i = 0; i < splits.size(); i++) { results[i] = new TableSnapshotRegionSplit(splits.get(i)); } return results; }
@Override public InputSplit[] getSplits(JobConf job, int numSplits) throws IOException { List<TableSnapshotInputFormatImpl.InputSplit> splits = TableSnapshotInputFormatImpl.getSplits(job); InputSplit[] results = new InputSplit[splits.size()]; for (int i = 0; i < splits.size(); i++) { results[i] = new TableSnapshotRegionSplit(splits.get(i)); } return results; }
public static List<InputSplit> getSplits(Configuration conf) throws IOException { String snapshotName = getSnapshotName(conf); Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = rootDir.getFileSystem(conf); SnapshotManifest manifest = getSnapshotManifest(conf, snapshotName, rootDir, fs); List<HRegionInfo> regionInfos = getRegionInfosFromManifest(manifest); // TODO: mapred does not support scan as input API. Work around for now. Scan scan = extractScanFromConf(conf); // the temp dir where the snapshot is restored Path restoreDir = new Path(conf.get(RESTORE_DIR_KEY)); return getSplits(scan, manifest, regionInfos, restoreDir, conf); }
public static List<InputSplit> getSplits(Configuration conf) throws IOException { String snapshotName = getSnapshotName(conf); Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = rootDir.getFileSystem(conf); SnapshotManifest manifest = getSnapshotManifest(conf, snapshotName, rootDir, fs); List<HRegionInfo> regionInfos = getRegionInfosFromManifest(manifest); // TODO: mapred does not support scan as input API. Work around for now. Scan scan = extractScanFromConf(conf); // the temp dir where the snapshot is restored Path restoreDir = new Path(conf.get(RESTORE_DIR_KEY)); RegionSplitter.SplitAlgorithm splitAlgo = getSplitAlgo(conf); int numSplits = conf.getInt(NUM_SPLITS_PER_REGION, 1); return getSplits(scan, manifest, regionInfos, restoreDir, conf, splitAlgo, numSplits); }
public static List<InputSplit> getSplits(Configuration conf) throws IOException { String snapshotName = getSnapshotName(conf); Path rootDir = FSUtils.getRootDir(conf); FileSystem fs = rootDir.getFileSystem(conf); SnapshotManifest manifest = getSnapshotManifest(conf, snapshotName, rootDir, fs); List<HRegionInfo> regionInfos = getRegionInfosFromManifest(manifest); // TODO: mapred does not support scan as input API. Work around for now. Scan scan = extractScanFromConf(conf); // the temp dir where the snapshot is restored Path restoreDir = new Path(conf.get(RESTORE_DIR_KEY)); RegionSplitter.SplitAlgorithm splitAlgo = getSplitAlgo(conf); int numSplits = conf.getInt(NUM_SPLITS_PER_REGION, 1); return getSplits(scan, manifest, regionInfos, restoreDir, conf, splitAlgo, numSplits); }
TableSnapshotInputFormatImpl.getSplits(scan, manifest, regionInfos, restoreDir, conf); rtn.addAll(splits);
TableSnapshotInputFormatImpl.getSplits(scan, manifest, regionInfos, restoreDir, conf); rtn.addAll(splits);
TableSnapshotInputFormatImpl.getSplits(scan, manifest, regionInfos, restoreDir, conf); rtn.addAll(splits);
public static List<InputSplit> getSplits(Scan scan, SnapshotManifest manifest, List<HRegionInfo> regionManifests, Path restoreDir, Configuration conf, RegionSplitter.SplitAlgorithm sa, int numSplits) throws IOException { // load table descriptor TableDescriptor htd = manifest.getTableDescriptor(); Path tableDir = FSUtils.getTableDir(restoreDir, htd.getTableName()); boolean localityEnabled = conf.getBoolean(SNAPSHOT_INPUTFORMAT_LOCALITY_ENABLED_KEY, SNAPSHOT_INPUTFORMAT_LOCALITY_ENABLED_DEFAULT); List<InputSplit> splits = new ArrayList<>(); for (HRegionInfo hri : regionManifests) { // load region descriptor if (numSplits > 1) { byte[][] sp = sa.split(hri.getStartKey(), hri.getEndKey(), numSplits, true); for (int i = 0; i < sp.length - 1; i++) { if (PrivateCellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(), sp[i], sp[i + 1])) { List<String> hosts = calculateLocationsForInputSplit(conf, htd, hri, tableDir, localityEnabled); Scan boundedScan = new Scan(scan); boundedScan.setStartRow(sp[i]); boundedScan.setStopRow(sp[i + 1]); splits.add(new InputSplit(htd, hri, hosts, boundedScan, restoreDir)); } }
public static List<InputSplit> getSplits(Scan scan, SnapshotManifest manifest, List<HRegionInfo> regionManifests, Path restoreDir, Configuration conf, RegionSplitter.SplitAlgorithm sa, int numSplits) throws IOException { // load table descriptor TableDescriptor htd = manifest.getTableDescriptor(); Path tableDir = FSUtils.getTableDir(restoreDir, htd.getTableName()); boolean localityEnabled = conf.getBoolean(SNAPSHOT_INPUTFORMAT_LOCALITY_ENABLED_KEY, SNAPSHOT_INPUTFORMAT_LOCALITY_ENABLED_DEFAULT); List<InputSplit> splits = new ArrayList<>(); for (HRegionInfo hri : regionManifests) { // load region descriptor if (numSplits > 1) { byte[][] sp = sa.split(hri.getStartKey(), hri.getEndKey(), numSplits, true); for (int i = 0; i < sp.length - 1; i++) { if (PrivateCellUtil.overlappingKeys(scan.getStartRow(), scan.getStopRow(), sp[i], sp[i + 1])) { List<String> hosts = calculateLocationsForInputSplit(conf, htd, hri, tableDir, localityEnabled); Scan boundedScan = new Scan(scan); if (scan.getStartRow().length == 0) { boundedScan.withStartRow(sp[i]); } else { boundedScan.withStartRow( Bytes.compareTo(scan.getStartRow(), sp[i]) > 0 ? scan.getStartRow() : sp[i]); }