@Override public Path getPathWithoutSchemeAndAuthority(Path path) { return Path.getPathWithoutSchemeAndAuthority(path); }
private static <T> Map<Path, T> populateNewT(Map<Path, T> pathToPartitionInfo) { Map<Path, T> newPathToPartitionInfo = new HashMap<>(); for (Map.Entry<Path, T> entry: pathToPartitionInfo.entrySet()) { T partDesc = entry.getValue(); Path pathOnly = Path.getPathWithoutSchemeAndAuthority(entry.getKey()); newPathToPartitionInfo.put(pathOnly, partDesc); } return newPathToPartitionInfo; }
private static Map<Path, PartitionDesc> populateNewPartitionDesc(Map<Path, PartitionDesc> pathToPartitionInfo) { Map<Path, PartitionDesc> newPathToPartitionInfo = new HashMap<>(); for (Map.Entry<Path, PartitionDesc> entry: pathToPartitionInfo.entrySet()) { PartitionDesc partDesc = entry.getValue(); Path pathOnly = Path.getPathWithoutSchemeAndAuthority(entry.getKey()); newPathToPartitionInfo.put(pathOnly, partDesc); } return newPathToPartitionInfo; }
protected static PartitionDesc getPartitionDescFromPath( Map<Path, PartitionDesc> pathToPartitionInfo, Path dir) throws IOException { PartitionDesc partDesc = pathToPartitionInfo.get(dir); if (partDesc == null) { // Note: we could call HiveFileFormatUtils.getPartitionDescFromPathRecursively for MM tables. // The recursive call is usually needed for non-MM tables, because the path management // is not strict and the code does whatever. That should not happen for MM tables. // Keep it like this for now; may need replacement if we find a valid use case. partDesc = pathToPartitionInfo.get(Path.getPathWithoutSchemeAndAuthority(dir)); } if (partDesc == null) { throw new IOException("cannot find dir = " + dir.toString() + " in " + pathToPartitionInfo); } return partDesc; }
protected static PartitionDesc getPartitionDescFromPath( Map<Path, PartitionDesc> pathToPartitionInfo, Path dir) throws IOException { PartitionDesc partDesc = pathToPartitionInfo.get(dir); if (partDesc == null) { partDesc = pathToPartitionInfo.get(Path.getPathWithoutSchemeAndAuthority(dir)); } if (partDesc == null) { throw new IOException("cannot find dir = " + dir.toString() + " in " + pathToPartitionInfo); } return partDesc; }
Map<Path, ArrayList<String>> removeScheme(Map<Path, ArrayList<String>> pathToAliases) { Map<Path, ArrayList<String>> result = new HashMap<>(); for (Map.Entry <Path, ArrayList<String>> entry : pathToAliases.entrySet()) { Path newKey = Path.getPathWithoutSchemeAndAuthority(entry.getKey()); StringInternUtils.internUriStringsInPath(newKey); result.put(newKey, entry.getValue()); } return result; }
final protected Path pushdownPath(String resPath) { Path p = new Path(pushdownRootPath() + resPath); return Path.getPathWithoutSchemeAndAuthority(p); }
Map<Path, ArrayList<String>> removeScheme(Map<Path, ArrayList<String>> pathToAliases) { Map<Path, ArrayList<String>> result = new HashMap<>(); for (Map.Entry <Path, ArrayList<String>> entry : pathToAliases.entrySet()) { Path newKey = Path.getPathWithoutSchemeAndAuthority(entry.getKey()); StringInternUtils.internUriStringsInPath(newKey); result.put(newKey, entry.getValue()); } return result; }
@Override public boolean apply(FileSystemDatasetVersion version) { Set<Path> paths = version.getPaths(); for (Path path : paths) { Path p = path.getPathWithoutSchemeAndAuthority(path); if (isPathHidden(p)) { return false; } } return true; } };
private boolean pathIsInPartition(Path split, Path partitionPath) { boolean schemeless = split.toUri().getScheme() == null; if (schemeless) { Path pathNoSchema = Path.getPathWithoutSchemeAndAuthority(partitionPath); return FileUtils.isPathWithinSubtree(split,pathNoSchema); } else { return FileUtils.isPathWithinSubtree(split,partitionPath); } }
public static String makeQualifiedPathInHBaseCluster(String inPath) { Path path = new Path(inPath); path = Path.getPathWithoutSchemeAndAuthority(path); FileSystem fs = HadoopUtil.getFileSystem(path, getCurrentHBaseConfiguration()); // Must be HBase's FS, not working FS return fs.makeQualified(path).toString(); }
private boolean pathIsInPartition(Path split, Path partitionPath) { boolean schemeless = split.toUri().getScheme() == null; if (schemeless) { Path pathNoSchema = Path.getPathWithoutSchemeAndAuthority(partitionPath); return FileUtils.isPathWithinSubtree(split,pathNoSchema); } else { return FileUtils.isPathWithinSubtree(split,partitionPath); } }
private Path getTrashPartitionLocation() { Preconditions.checkArgument(this.state.contains(ComplianceConfigurationKeys.TRASH_DIR), "Missing required property " + ComplianceConfigurationKeys.TRASH_DIR); return new Path(StringUtils.join(Arrays.asList(this.state.getProp(ComplianceConfigurationKeys.TRASH_DIR), Path.getPathWithoutSchemeAndAuthority(getLocation()).toString()), '/')); }
private String getJdbcHdfsWorkingDirectory() { if (StringUtils.isNotEmpty(getJdbcFileSystem())) { Path workingDir = new Path(getReadHdfsWorkingDirectory()); return new Path(getJdbcFileSystem(), Path.getPathWithoutSchemeAndAuthority(workingDir)).toString() + "/"; } return getReadHdfsWorkingDirectory(); }
public String getReadHdfsWorkingDirectory() { if (StringUtils.isNotEmpty(getHBaseClusterFs())) { Path workingDir = new Path(getHdfsWorkingDirectory()); return new Path(getHBaseClusterFs(), Path.getPathWithoutSchemeAndAuthority(workingDir)).toString() + "/"; } return getHdfsWorkingDirectory(); }
private Path getNewVersionLocation() { Preconditions.checkArgument(this.state.contains(ComplianceConfigurationKeys.BACKUP_DIR), "Missing required property " + ComplianceConfigurationKeys.BACKUP_DIR); HivePartitionRetentionVersion version = (HivePartitionRetentionVersion) this.datasetVersion; if (PartitionUtils.isUnixTimeStamp(version.getLocation().getName())) { return new Path(StringUtils.join(Arrays.asList(this.state.getProp(ComplianceConfigurationKeys.BACKUP_DIR), Path.getPathWithoutSchemeAndAuthority(version.getLocation().getParent()).toString(), version.getTimeStamp()), '/')); } else { return new Path(StringUtils.join(Arrays.asList(this.state.getProp(ComplianceConfigurationKeys.BACKUP_DIR), Path.getPathWithoutSchemeAndAuthority(version.getLocation()).toString(), version.getTimeStamp()), '/')); } }
/** * Sets the mapWork variable based on the current JobConf in order to get all partitions. * * @param job */ private void updateMrWork(final JobConf job) { final String plan = HiveConf.getVar(job, HiveConf.ConfVars.PLAN); if (mapWork == null && plan != null && plan.length() > 0) { mapWork = Utilities.getMapWork(job); pathToPartitionInfo.clear(); for (final Map.Entry<Path, PartitionDesc> entry : mapWork.getPathToPartitionInfo().entrySet()) { // key contains scheme (such as pfile://) and we want only the path portion fix in HIVE-6366 pathToPartitionInfo.put(StringInternUtils.internUriStringsInPath( Path.getPathWithoutSchemeAndAuthority(entry.getKey())), entry.getValue()); } } }
@Override protected void initInternal() throws IOException { Map<FileStatus, FileSystem> fileStatusConfMap = new LinkedHashMap<>(); for (ReadEntryWithPath entry : entries) { Path path = new Path(entry.getPath()); Configuration conf = new ProjectionPusher().pushProjectionsAndFilters( new JobConf(hiveStoragePlugin.getHiveConf()), path.getParent()); FileSystem fs = path.getFileSystem(conf); fileStatusConfMap.put(fs.getFileStatus(Path.getPathWithoutSchemeAndAuthority(path)), fs); } parquetTableMetadata = Metadata.getParquetTableMetadata(fileStatusConfMap, readerConfig); }
/** * Paths can be qualified or not. This does the assertion using String->Path conversion. */ private void assertPathListsEqual(List<String> list1, List<String> list2) { List<Path> l1 = new ArrayList<>(list1.size()); for (String path : list1) { l1.add(Path.getPathWithoutSchemeAndAuthority(new Path(path))); } List<Path> l2 = new ArrayList<>(list2.size()); for (String path : list2) { l2.add(Path.getPathWithoutSchemeAndAuthority(new Path(path))); } assertEquals(l1, l2); }
private List<String> listDir(Path p) throws IOException { ArrayList<String> result = new ArrayList<>(); RemoteIterator<LocatedFileStatus> fileNames = fs.listFiles(p, false); while (fileNames.hasNext()) { LocatedFileStatus fileStatus = fileNames.next(); result.add(Path.getPathWithoutSchemeAndAuthority(fileStatus.getPath()).toString()); } return result; }