public InstrumentedFileSystem(String scheme, FileSystem underlyingFileSystem) { super(scheme, underlyingFileSystem.getScheme()); this.underlyingFs = underlyingFileSystem; }
private static boolean isS3(FileSystem fs) { try { return "s3a".equalsIgnoreCase(fs.getScheme()); } catch (UnsupportedOperationException ex) { // Some FS-es do not implement getScheme, e.g. ProxyLocalFileSystem. return false; } }
private boolean isLocal(FileSystem fs) { return fs.getScheme().equals("file"); }
/** * Check if the FileSystem is a ViewFileSystem. * * @param fileSystem * @return true if the fileSystem is ViewFileSystem */ public static boolean isViewFileSystem(final FileSystem fileSystem) { return fileSystem.getScheme().equals(FsConstants.VIEWFS_SCHEME); }
static boolean isHdfs(FileSystem fs) { return fs.getScheme().equals("hdfs"); }
private void dropPathAndUnregisterDeleteOnExit(Path path, Configuration conf, boolean localFs) { FileSystem fs = null; try { if (localFs) { fs = FileSystem.getLocal(conf); } else { fs = path.getFileSystem(conf); } fs.cancelDeleteOnExit(path); fs.delete(path, true); LOG.info("Deleted directory: {} on fs with scheme {}", path, fs.getScheme()); } catch (IllegalArgumentException | UnsupportedOperationException | IOException e) { LOG.error("Failed to delete path at {} on fs with scheme {}", path, (fs == null ? "Unknown-null" : fs.getScheme()), e); } }
protected MultipartUploader createMultipartUploader(FileSystem fs, Configuration conf) { if (fs.getScheme().equals("file")) { return new FileSystemMultipartUploader(fs); } return null; } }
public static boolean isBlobStorageFileSystem(final Configuration conf, final FileSystem fs) { return fs != null && isBlobStorageScheme(conf, fs.getScheme()); }
/** * Creates a new Recoverable writer. * @param fs The Hadoop file system on which the writer operates. */ public HadoopRecoverableWriter(org.apache.hadoop.fs.FileSystem fs) { this.fs = checkNotNull(fs); // This writer is only supported on a subset of file systems, and on // specific versions. We check these schemes and versions eagerly for // better error messages. if (!"hdfs".equalsIgnoreCase(fs.getScheme()) || !HadoopUtils.isMinHadoopVersion(2, 7)) { throw new UnsupportedOperationException( "Recoverable writers on Hadoop are only supported for HDFS and for Hadoop version 2.7 or newer"); } }
@Override protected FileSystem getDefaultStoreFs(Config factoryConfig, Optional<URI> configDefinedDefaultURI) { try { if (configDefinedDefaultURI.isPresent() && configDefinedDefaultURI.get().getAuthority() != null) { return FileSystem.get(configDefinedDefaultURI.get(), new Configuration()); } else { FileSystem fs = FileSystem.get(new Configuration()); return HDFS_SCHEME_NAME.equals(fs.getScheme()) ? fs : null; } } catch (IOException ioe) { throw new RuntimeException("Could not create default store fs for scheme " + getScheme()); } }
private void dropPathAndUnregisterDeleteOnExit(Path path, Configuration conf, boolean localFs) { FileSystem fs = null; try { if (localFs) { fs = FileSystem.getLocal(conf); } else { fs = path.getFileSystem(conf); } fs.cancelDeleteOnExit(path); fs.delete(path, true); LOG.info("Deleted directory: {} on fs with scheme {}", path, fs.getScheme()); } catch (IOException e) { LOG.error("Failed to delete path at {} on fs with scheme {}", path, (fs == null ? "Unknown-null" : fs.getScheme()), e); } }
public static Path fullyQualifiedHDFSUri(Path input, FileSystem hdfsFileSystem) throws SemanticException { URI uri = input.toUri(); String scheme = hdfsFileSystem.getScheme(); String authority = hdfsFileSystem.getUri().getAuthority(); String path = uri.getPath(); try { return new Path(new URI(scheme, authority, path, null, null)); } catch (URISyntaxException e) { throw new SemanticException(ErrorMsg.INVALID_PATH.getMsg(), e); } } }
private boolean ifEnableMoveOptimization(Path filePath, org.apache.hadoop.conf.Configuration conf) throws Exception { if (filePath == null) { throw new HiveException("filePath cannot be null"); } URI uri = filePath.toUri(); String scheme = uri.getScheme(); scheme = StringUtils.isBlank(scheme) ? FileSystem.get(uri, conf).getScheme() : scheme; if (StringUtils.isBlank(scheme)) { throw new HiveException("Cannot get valid scheme for " + filePath); } LOG.info("scheme is " + scheme); String[] schmeList = conf.get(REPL_MOVE_OPTIMIZED_FILE_SCHEMES.varname).toLowerCase().split(","); for (String schemeIter : schmeList) { if (schemeIter.trim().equalsIgnoreCase(scheme.trim())) { return true; } } return false; }
@Override public Descriptor getDataDescriptor() { // Dataset is resulted from WriterUtils.getWriterOutputDir(properties, this.numBranches, this.branchId) // The writer dataset might not be same as the published dataset DatasetDescriptor datasetDescriptor = new DatasetDescriptor(fs.getScheme(), outputFile.getParent().toString()); if (partitionKey == null) { return datasetDescriptor; } return new PartitionDescriptor(partitionKey, datasetDescriptor); }
/** * Create destination dataset descriptor */ protected DatasetDescriptor createDestinationDescriptor(WorkUnitState state, int branchId) { Path publisherOutputDir = getPublisherOutputDir(state, branchId); FileSystem fs = this.publisherFileSystemByBranches.get(branchId); DatasetDescriptor destination = new DatasetDescriptor(fs.getScheme(), publisherOutputDir.toString()); destination.addMetadata(DatasetConstants.FS_URI, fs.getUri().toString()); destination.addMetadata(DatasetConstants.BRANCH, String.valueOf(branchId)); return destination; }
/** * Set file system based source and destination dataset for this {@link CopyableFile} * * @param originFs {@link FileSystem} where this {@link CopyableFile} origins * @param targetFs {@link FileSystem} where this {@link CopyableFile} is copied to */ public void setFsDatasets(FileSystem originFs, FileSystem targetFs) { /* * By default, the raw Gobblin dataset for CopyableFile lineage is its parent folder * if itself is not a folder */ boolean isDir = origin.isDirectory(); Path fullSourcePath = Path.getPathWithoutSchemeAndAuthority(origin.getPath()); String sourceDatasetName = isDir ? fullSourcePath.toString() : fullSourcePath.getParent().toString(); DatasetDescriptor sourceDataset = new DatasetDescriptor(originFs.getScheme(), sourceDatasetName); sourceDataset.addMetadata(DatasetConstants.FS_URI, originFs.getUri().toString()); sourceData = sourceDataset; Path fullDestinationPath = Path.getPathWithoutSchemeAndAuthority(destination); String destinationDatasetName = isDir ? fullDestinationPath.toString() : fullDestinationPath.getParent().toString(); DatasetDescriptor destinationDataset = new DatasetDescriptor(targetFs.getScheme(), destinationDatasetName); destinationDataset.addMetadata(DatasetConstants.FS_URI, targetFs.getUri().toString()); destinationData = destinationDataset; }
private DatasetDescriptor createSourceDataset() { try { String sourceTable = getTable().getDbName() + "." + getTable().getTableName(); DatasetDescriptor source = new DatasetDescriptor(DatasetConstants.PLATFORM_HIVE, sourceTable); Path sourcePath = getTable().getDataLocation(); log.info(String.format("[%s]Source path %s being used in conversion", this.getClass().getName(), sourcePath)); String sourceLocation = Path.getPathWithoutSchemeAndAuthority(sourcePath).toString(); FileSystem sourceFs = sourcePath.getFileSystem(new Configuration()); source.addMetadata(DatasetConstants.FS_SCHEME, sourceFs.getScheme()); source.addMetadata(DatasetConstants.FS_LOCATION, sourceLocation); return source; } catch (IOException e) { throw new RuntimeException(e); } }
@Test public void testValidAndInvalidFileSystems() { FileSystem fs = mock(FileSystem.class); /* Valid FileSystem schemes */ doReturn("s3a").when(fs).getScheme(); assertTrue(isBlobStorageFileSystem(conf, fs)); doReturn("swift").when(fs).getScheme(); assertTrue(isBlobStorageFileSystem(conf, fs)); /* Invalid FileSystem schemes */ doReturn("hdfs").when(fs).getScheme(); assertFalse(isBlobStorageFileSystem(conf, fs)); doReturn("").when(fs).getScheme(); assertFalse(isBlobStorageFileSystem(conf, fs)); assertFalse(isBlobStorageFileSystem(conf, null)); }
@Test public void createDestinationPath() throws IOException, SemanticException, URISyntaxException { mockStatic(FileSystem.class); when(FileSystem.get(any(Configuration.class))).thenReturn(mockFs); when(FileSystem.get(any(URI.class), any(Configuration.class))).thenReturn(mockFs); when(mockFs.getScheme()).thenReturn("hdfs"); when(mockFs.getUri()).thenReturn(new URI("hdfs", "somehost:9000", null, null, null)); mockStatic(System.class); when(System.nanoTime()).thenReturn(Long.MAX_VALUE); when(functionObj.getFunctionName()).thenReturn("someFunctionName"); mockStatic(ReplCopyTask.class); Task mock = mock(Task.class); when(ReplCopyTask.getLoadCopyTask(any(ReplicationSpec.class), any(Path.class), any(Path.class), any(HiveConf.class))).thenReturn(mock); ResourceUri resourceUri = function.destinationResourceUri(new ResourceUri(ResourceType.JAR, "hdfs://localhost:9000/user/someplace/ab.jar#e094828883")); assertThat(resourceUri.getUri(), is(equalTo( "hdfs://somehost:9000/someBasePath/withADir/replicadbname/somefunctionname/" + String .valueOf(Long.MAX_VALUE) + "/ab.jar"))); } }