@Override public void setup(List<String> columnNames) throws IOException { this.columnNames = columnNames; this.fs = FileSystemWrapper.get(conf, stats); }
@Override public boolean containerExists(NamespaceKey key) { List<String> folderPath = key.getPathComponents(); try { return getFS(SystemUser.SYSTEM_USERNAME).exists(PathUtils.toFSPath( ImmutableList.<String>builder() .addAll(folderPath.subList(1, folderPath.size())) .build())); }catch(IOException e) { logger.info("IOException while trying to retrieve home files.", e); return false; } }
public static FileSystemWrapper get(Configuration fsConf, OperatorStats stats) throws IOException { return new FileSystemWrapper(fsConf, stats, null); }
@Override public FileSystem getFileSystem(Configuration conf) throws IOException { // Do not return the original filesystem as-is because: // - the path might have been canonicalized and might point to a different URI (viewFS) // - this method might be called from a different security context (and the filesystem // should reflect this) // Instead recreate a new wrapper (but keep the stats context) return FileSystemWrapper.get(this.toUri(), conf, fs.getOperatorStats()); } }
@Override public Path canonicalizePath(Path p) throws IOException { try { Path cp = canonicalizePath(underlyingFs, p); return new PathWrapperWithFileSystem(cp, this); } catch(FSError e) { throw propagateFSError(e); } }
public FileSystemWrapper getFilesystemAndCreatePaths(String hostname) throws IOException { FileSystemWrapper fs = FileSystemWrapper.get(uri.get(), new Configuration()); fs.mkdirs(getPath(), HomeFileSystemStoragePlugin.DEFAULT_PERMISSIONS); fs.mkdirs(getInnerUploads(), HomeFileSystemStoragePlugin.DEFAULT_PERMISSIONS); if(hostname == null) { hostname = this.hostname; } if(hostname != null) { fs.mkdirs(getStagingPath(hostname), HomeFileSystemStoragePlugin.DEFAULT_PERMISSIONS); fs.deleteOnExit(getStagingPath(hostname)); } return fs; }
private FileSystemPartitionUpdateKey getFSBasedUpdateKey(String partitionDir, JobConf job, boolean isRecursive, int partitionId) throws IOException { final List<FileSystemCachedEntity> cachedEntities = Lists.newArrayList(); final Path rootLocation = new Path(partitionDir); final FileSystemWrapper fs = FileSystemWrapper.get(rootLocation, job); if (fs.exists(rootLocation)) { final FileStatus rootStatus = fs.getFileStatus(rootLocation); if (rootStatus.isDirectory()) { cachedEntities.add(FileSystemCachedEntity.newBuilder() .build()); List<FileStatus> statuses = isRecursive ? fs.listRecursive(rootLocation, false) : fs.list(rootLocation, false); for (FileStatus fileStatus : statuses) { final Path filePath = fileStatus.getPath(); .setPartitionRootDir(fs.makeQualified(rootLocation).toString()) .addAllCachedEntities(cachedEntities) .build();
private boolean hasChanged() throws IOException { final JobConf jobConf = new JobConf(hiveConf); for (Prop prop : HiveReaderProtoUtil.getPartitionProperties(tableXattr, updateKey.getPartitionId())) { jobConf.set(prop.getKey(), prop.getValue()); } Preconditions.checkArgument(updateKey.getCachedEntitiesCount() > 0, "hive partition update key should contain at least one path"); // create filesystem based on the first path which is root of the partition directory. final FileSystemWrapper fs = FileSystemWrapper.get(new Path(updateKey.getPartitionRootDir()), jobConf); for (FileSystemCachedEntity cachedEntity : updateKey.getCachedEntitiesList()) { final Path cachedEntityPath; if (cachedEntity.getPath() == null || cachedEntity.getPath().isEmpty()) { cachedEntityPath = new Path(updateKey.getPartitionRootDir()); } else { cachedEntityPath = new Path(updateKey.getPartitionRootDir(), cachedEntity.getPath()); } if (fs.exists(cachedEntityPath)) { final FileStatus fileStatus = fs.getFileStatus(cachedEntityPath); if (cachedEntity.getLastModificationTime() < fileStatus.getModificationTime()) { return true; } } else { return true; } } return false; } }
@Override public void setup(VectorAccessible incoming, OutputEntryListener outputEntryListener, WriteStatsListener writeStatsListener) throws IOException { Preconditions.checkArgument(incoming.getSchema().getSelectionVectorMode() == SelectionVectorMode.NONE, "SelectionVector remover is not supported."); this.incoming = incoming; this.outputEntryListener = outputEntryListener; this.writeStatsListener = writeStatsListener; this.fs = FileSystemWrapper.get(location, writerConfig.getFsConf(), stats); this.currentFile = fs.canonicalizePath(new Path(location, String.format("%s_%d.%s", prefix, nextFileIndex, extension))); this.relativePath = currentFile.getName(); this.currentFileOutputStream = fs.create(currentFile); listOfFilesCreated.add(currentFile); // write magic word bytes currentFileOutputStream.write(MAGIC_STRING.getBytes()); for(final VectorWrapper<? extends ValueVector> vw : incoming) { Preconditions.checkArgument(!vw.isHyper(), "Writing hyper vectors to arrow format is not supported."); footerBuilder.addField(TypeHelper.getMetadata(vw.getValueVector())); } nextFileIndex++; recordCount = 0; }
current = fs.getFileStatus(status.getSymlink()); try (FSDataInputStream is = fs.open(status.getPath())) { for(RangeMagics rMagic : ranges) { Range<Long> r = rMagic.range;
/** * Get a preview of the implicit field schema without moving to file level paths. * * @param selection * @return A list of name value pairs with null values. These should only be used for schema learning, not reading. * @throws IOException * @throws SchemaChangeException */ public List<NameValuePair<?>> getImplicitFieldsForSample(FileSelection selection) throws IOException, SchemaChangeException { final List<NameValuePair<?>> fields = new ArrayList<>(); FileStatus fileStatus = fs.getFileStatus(new Path(selection.getSelectionRoot())); if (enableDirsFields && fileStatus.isDirectory()) { int maxDepth = selection.getMaxDepth(fileStatus); for (int i = 0; i < maxDepth - 1; i++) { fields.add(new VarCharNameValuePair(partitionDesignator + i, "dir0")); } } if(enableFileField) { fields.add(new VarCharNameValuePair(fileDesignator, "/")); } if (enableModTimeField) { fields.add(new BigIntNameValuePair(modTimeDesignator, 0L)); } fields.add(new BigIntNameValuePair(IncrementalUpdateUtils.UPDATE_COLUMN, 0L)); return fields; }
/** * Create the supporting directory for this plugin if it doesn't yet exist. * @throws IOException */ private void createIfNecessary() throws IOException { if(!config.createIfMissing()) { return; } try { // no need to exists here as FileSystemWrapper does an exists check and this is a noop if already existing. fs.mkdirs(config.getPath()); } catch (IOException ex) { try { if(fs.exists(config.getPath())) { // race creation, ignore. return; } } catch (IOException existsFailure) { // we're doing the check above to detect a race condition. if we fail, ignore the failure and just fall through to throwing the originally caught exception. ex.addSuppressed(existsFailure); } throw new IOException(String.format("Failure to create directory %s.", config.getPath().toString()), ex); } }
@Override public void startPartition(WritePartition partition) throws Exception { if(this.partition != null){ close(); } this.partition = partition; // open a new file for writing data with new schema try { this.path = fs.canonicalizePath(partition.qualified(location, prefix + "_" + index + "." + extension)); fos = fs.create(path); stream = new PrintStream(fos); stream.write(ByteOrderMark.UTF_8.getBytes(), 0, ByteOrderMark.UTF_8.length()); logger.debug("Created file: {}", path); } catch (IOException e) { throw UserException.dataWriteError(e) .message("Failure while attempting to write file %s.", path) .build(logger); } index++; String columns = Joiner.on(fieldDelimiter).join(columnNames); stream.print(columns); stream.print(lineDelimiter); }
status = fs.getFileStatus(path); } catch(IOException ex) { RemoteIterator<LocatedFileStatus> iter = excludeHidden(fs.listFiles(path, true)); return getData( format,
@Override public boolean createOrUpdateView(NamespaceKey key, View view, SchemaConfig schemaConfig) throws IOException { if(!getMutability().hasMutationCapability(MutationType.VIEW, schemaConfig.isSystemUser())) { throw UserException.parseError() .message("Unable to create view. Schema [%s] is immutable for this user.", key.getParent()) .build(logger); } Path viewPath = getViewPath(key.getPathComponents()); FileSystemWrapper fs = getFS(schemaConfig.getUserName()); boolean replaced = fs.exists(viewPath); final FsPermission viewPerms = new FsPermission(schemaConfig.getOption(ExecConstants.NEW_VIEW_DEFAULT_PERMS_KEY).getStringVal()); try (OutputStream stream = FileSystemWrapper.create(fs, viewPath, viewPerms)) { lpPersistance.getMapper().writeValue(stream, view); } return replaced; }
@Test public void testReadIOStats() throws Exception { FileSystemWrapper dfs = null; InputStream is = null; Configuration conf = new Configuration(); conf.set(FileSystem.FS_DEFAULT_NAME_KEY, "file:///"); OpProfileDef profileDef = new OpProfileDef(0 /*operatorId*/, 0 /*operatorType*/, 0 /*inputCount*/); OperatorStats stats = new OperatorStats(profileDef, null /*allocator*/); // start wait time method in OperatorStats expects the OperatorStats state to be in "processing" stats.startProcessing(); try { dfs = new FileSystemWrapper(conf, stats, null); is = dfs.open(new Path(tempFilePath)); byte[] buf = new byte[8000]; while (is.read(buf, 0, buf.length) != -1) { } } finally { stats.stopProcessing(); if (is != null) { is.close(); } if (dfs != null) { dfs.close(); } } OperatorProfile operatorProfile = stats.getProfile(); assertTrue("Expected wait time is non-zero, but got zero wait time", operatorProfile.getWaitNanos() > 0); }
dfs = new FileSystemWrapper(conf, stats, null); os = dfs.create(new Path(tempFolder.getRoot().getPath(), "dremioFSWriteTest.txt")); dfs.close();
@Override public RecordReader getRecordReader(OperatorContext context, FileSystemWrapper dfs, EasyDatasetSplitXAttr splitAttributes, List<SchemaPath> columns) throws ExecutionSetupException { final Path path = dfs.makeQualified(new Path(splitAttributes.getPath())); final ExcelFormatPluginConfig excelFormatConfig = (ExcelFormatPluginConfig) formatConfig; return new ExcelRecordReader( context, dfs, path, excelFormatConfig, columns); }
public FileSystemWrapper(Configuration fsConf, FileSystem fs, OperatorStats operatorStats) { this.underlyingFs = fs; this.codecFactory = new CompressionCodecFactory(fsConf); this.operatorStats = operatorStats; this.isPdfs = (underlyingFs instanceof PathCanonicalizer); // only pdfs implements PathCanonicalizer this.isMapRfs = isMapRfs(underlyingFs); }