public SubsetFilesCopyableDataset(final FileSystem fs, Path rootPath, Properties properties, String idenifier, List<FileStatus> subFiles) { this.rootPath = PathUtils.getPathWithoutSchemeAndAuthority(rootPath); this.fs = fs; this.files = subFiles; this.identifier = idenifier; this.props = properties; }
/** * Compare two path without shedme and authority (the prefix) * @param path1 * @param path2 * @return */ public static boolean compareWithoutSchemeAndAuthority(Path path1, Path path2) { return PathUtils.getPathWithoutSchemeAndAuthority(path1).equals(getPathWithoutSchemeAndAuthority(path2)); } }
private Path getJobPath(Properties jobProps) { return PathUtils.getPathWithoutSchemeAndAuthority(new Path(jobProps.getProperty(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY))); }
public static Path relativizePath(Path fullPath, Path pathPrefix) { return new Path(getPathWithoutSchemeAndAuthority(pathPrefix).toUri() .relativize(getPathWithoutSchemeAndAuthority(fullPath).toUri())); }
public RecursivePathFinder(final FileSystem fs, Path rootPath, Properties properties) { this.rootPath = PathUtils.getPathWithoutSchemeAndAuthority(rootPath); this.fs = fs; this.pathFilter = DatasetUtils.instantiatePathFilter(properties); }
/** * @param testTempDirPath under which all test files are created on the FileSystem * @param testSetupConfPath setup config file path in classpath */ public RetentionTestDataGenerator(Path testTempDirPath, Path testSetupConfPath, FileSystem fs) { this.fs = fs; this.testTempDirPath = testTempDirPath; this.setupConfig = ConfigFactory.parseResources(PathUtils.getPathWithoutSchemeAndAuthority(testSetupConfPath).toString()); if (!this.setupConfig.hasPath(DATA_GENERATOR_KEY)) { throw new RuntimeException(String.format("Failed to load setup config at %s", testSetupConfPath.toString())); } }
private void unscheduleJobAtPath(Path path) { try { Path pathWithoutSchemeOrAuthority = PathUtils.getPathWithoutSchemeAndAuthority(path); String jobName = this.jobNameMap.get(pathWithoutSchemeOrAuthority); if (jobName == null) { LOG.info("Could not find a scheduled job to unschedule with path " + pathWithoutSchemeOrAuthority); return; } LOG.info("Unscheduling job " + jobName); this.jobScheduler.unscheduleJob(jobName); this.jobNameMap.remove(pathWithoutSchemeOrAuthority); } catch (JobException je) { LOG.error("Could not unschedule job " + this.jobNameMap.get(path)); } }
public RecursiveCopyableDataset(final FileSystem fs, Path rootPath, Properties properties, Path glob) { this.rootPath = PathUtils.getPathWithoutSchemeAndAuthority(rootPath); this.fs = fs; this.pathFilter = DatasetUtils.instantiatePathFilter(properties); this.copyableFileFilter = DatasetUtils.instantiateCopyableFileFilter(properties); this.glob = glob; this.update = Boolean.parseBoolean(properties.getProperty(UPDATE_KEY)); this.delete = Boolean.parseBoolean(properties.getProperty(DELETE_KEY)); this.deleteEmptyDirectories = Boolean.parseBoolean(properties.getProperty(DELETE_EMPTY_DIRECTORIES_KEY)); this.properties = properties; }
protected static Path replacedPrefix(Path sourcePath, Path prefixTobeReplaced, Path prefixReplacement) { Path sourcePathWithoutSchemeAndAuthority = PathUtils.getPathWithoutSchemeAndAuthority(sourcePath); Preconditions.checkArgument(PathUtils.isAncestor(prefixTobeReplaced, sourcePathWithoutSchemeAndAuthority), "When replacing prefix, all locations must be descendants of the prefix. " + "The prefix: %s, file location: %s.", prefixTobeReplaced, sourcePathWithoutSchemeAndAuthority); Path relativePath = PathUtils.relativizePath(sourcePathWithoutSchemeAndAuthority, prefixTobeReplaced); Path result = new Path(prefixReplacement, relativePath); return result; }
/** * Finds all directories satisfying the input glob pattern, and creates a {@link gobblin.data.management.retention.dataset.CleanableDataset} * for each one using {@link #datasetAtPath}. * @return List of {@link gobblin.data.management.retention.dataset.CleanableDataset}s in the file system. * @throws IOException */ @Override public List<T> findDatasets() throws IOException { List<T> datasets = Lists.newArrayList(); LOG.info("Finding datasets for pattern " + this.datasetPattern); FileStatus[] fileStatuss = this.getDatasetDirs(); if (fileStatuss != null) { for (FileStatus fileStatus : fileStatuss) { Path pathToMatch = PathUtils.getPathWithoutSchemeAndAuthority(fileStatus.getPath()); if (this.blacklist.isPresent() && this.blacklist.get().matcher(pathToMatch.toString()).find()) { continue; } LOG.info("Found dataset at " + fileStatus.getPath()); datasets.add(datasetAtPath(PathUtils.getPathWithoutSchemeAndAuthority(fileStatus.getPath()))); } } return datasets; }
private Config loadHoconConfigAtPath(Path path) throws IOException { try (InputStream is = fs.open(path); Reader reader = new InputStreamReader(is, Charsets.UTF_8)) { return ConfigFactory.parseMap(ImmutableMap.of(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY, PathUtils.getPathWithoutSchemeAndAuthority(path).toString())) .withFallback(ConfigFactory.parseReader(reader, ConfigParseOptions.defaults().setSyntax(ConfigSyntax.CONF))); } }
/** * Deletes empty directories starting with startPath and all ancestors up to but not including limitPath. * @param fs {@link FileSystem} where paths are located. * @param limitPath only {@link Path}s that are strict descendants of this path will be deleted. * @param startPath first {@link Path} to delete. Afterwards empty ancestors will be deleted. * @throws IOException */ public static void deleteEmptyParentDirectories(FileSystem fs, Path limitPath, Path startPath) throws IOException { if (PathUtils.isAncestor(limitPath, startPath) && !PathUtils.getPathWithoutSchemeAndAuthority(limitPath) .equals(PathUtils.getPathWithoutSchemeAndAuthority(startPath)) && fs.listStatus(startPath).length == 0) { if (!fs.delete(startPath, false)) { log.warn("Failed to delete empty directory " + startPath); } else { log.info("Deleted empty directory " + startPath); } deleteEmptyParentDirectories(fs, limitPath, startPath.getParent()); } }
/** * Checks whether possibleAncestor is an ancestor of fullPath. * @param possibleAncestor Possible ancestor of fullPath. * @param fullPath path to check. * @return true if possibleAncestor is an ancestor of fullPath. */ public static boolean isAncestor(Path possibleAncestor, Path fullPath) { return !relativizePath(fullPath, possibleAncestor).equals(getPathWithoutSchemeAndAuthority(fullPath)); }
/** * Load a {@link Properties} compatible path using fallback as fallback. * @return The {@link Config} in path with fallback as fallback. * @throws IOException */ private Config loadJavaPropsWithFallback(Path propertiesPath, Config fallback) throws IOException { PropertiesConfiguration propertiesConfiguration = new PropertiesConfiguration(); try (InputStreamReader inputStreamReader = new InputStreamReader(this.fs.open(propertiesPath), Charsets.UTF_8)) { propertiesConfiguration.load(inputStreamReader); Config configFromProps = ConfigUtils.propertiesToConfig(ConfigurationConverter.getProperties(propertiesConfiguration)); return ConfigFactory.parseMap(ImmutableMap.of(ConfigurationKeys.JOB_CONFIG_FILE_PATH_KEY, PathUtils.getPathWithoutSchemeAndAuthority(propertiesPath).toString())) .withFallback(configFromProps) .withFallback(fallback); } catch (ConfigurationException ce) { throw new IOException(ce); } }
public static Path getOutputFilePath(CopyableFile file, Path outputDir, CopyEntity.DatasetAndPartition datasetAndPartition) { Path destinationWithoutSchemeAndAuthority = PathUtils.getPathWithoutSchemeAndAuthority(file.getDestination()); return new Path(getPartitionOutputRoot(outputDir, datasetAndPartition), PathUtils.withoutLeadingSeparator(destinationWithoutSchemeAndAuthority)); }
File file = new File(PathUtils.getPathWithoutSchemeAndAuthority(fullFilePath).toString()); boolean modifiedFile = file.setLastModified(FORMATTER.parseMillis(fileToCreate.getString(TEST_DATA_MOD_TIME_LOCAL_KEY)));
.withFallback(ConfigFactory.parseMap(ImmutableMap.of("testNameTempPath", PathUtils.getPathWithoutSchemeAndAuthority(testNameTempPath).toString()))).resolve();
@VisibleForTesting protected CopyableFile generateCopyableFile(FileStatus singleFile, Path targetPath, long timestampFromPath, Path locationToCopy) throws IOException { return CopyableFile.fromOriginAndDestination(srcFs, singleFile, targetPath, configuration) .originTimestamp(timestampFromPath).upstreamTimestamp(timestampFromPath) .fileSet(PathUtils.getPathWithoutSchemeAndAuthority(locationToCopy).toString()).build(); }
return targetFs.makeQualified(new Path(this.targetTableRoot.get(), relativePath)); } else { return targetFs.makeQualified(PathUtils.getPathWithoutSchemeAndAuthority(sourcePath));
for(FileStatus f: allFilesInTarget){ if(pathFilter.accept(f.getPath())){ copyToFileMap.put(PathUtils.getPathWithoutSchemeAndAuthority(f.getPath()), f); Path relative = PathUtils.relativizePath(PathUtils.getPathWithoutSchemeAndAuthority(originFileStatus.getPath()), PathUtils.getPathWithoutSchemeAndAuthority(copyFrom.getDatasetPath())); .fileSet(PathUtils.getPathWithoutSchemeAndAuthority(copyTo.getDatasetPath()).toString()).build());