public FileUpdateKey newMessage() { return new FileUpdateKey(); }
private Collection<FsPermissionTask> getUpdateKeyPermissionTasks(DatasetConfig datasetConfig, FileSystemWrapper userFs) { final FileUpdateKey fileUpdateKey = new FileUpdateKey(); ProtostuffIOUtil.mergeFrom(datasetConfig.getReadDefinition().getReadSignature().toByteArray(), fileUpdateKey, FileUpdateKey.getSchema()); if (fileUpdateKey.getCachedEntitiesList() == null || fileUpdateKey.getCachedEntitiesList().isEmpty()) { return Collections.emptyList(); } final List<FsPermissionTask> fsPermissionTasks = Lists.newArrayList(); final FsAction action; final List<Path> batch = Lists.newArrayList(); //DX-7850 : remove once solution for maprfs is found if (userFs.isMapRfs()) { action = FsAction.READ; } else { action = FsAction.READ_EXECUTE; } for (FileSystemCachedEntity cachedEntity : fileUpdateKey.getCachedEntitiesList()) { batch.add(new Path(cachedEntity.getPath())); if (batch.size() == PERMISSION_CHECK_TASK_BATCH_SIZE) { // make a copy of batch fsPermissionTasks.add(new FsPermissionTask(userFs, Lists.newArrayList(batch), action)); batch.clear(); } } if (!batch.isEmpty()) { fsPermissionTasks.add(new FsPermissionTask(userFs, batch, action)); } return fsPermissionTasks; }
private SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig oldConfig, FormatPlugin formatPlugin, FileSystemWrapper fs, FileConfig fileConfig) throws IOException { final List<FileSystemCachedEntity> cachedEntities = Lists.newArrayList(); final FileStatus rootStatus = fs.getFileStatus(new Path(fileConfig.getLocation())); final Path combined = new Path("/", PathUtils.removeLeadingSlash(fileConfig.getLocation())); final FileSelection fileSelection = FileSelection.create(fs, combined); if (fileSelection == null) { return null; } // first entity is always a root if (rootStatus.isDirectory()) { cachedEntities.add(fromFileStatus(rootStatus)); } for (FileStatus dirStatus: fileSelection.getAllDirectories()) { cachedEntities.add(fromFileStatus(dirStatus)); } if(cachedEntities.isEmpty()){ // this is a single file. cachedEntities.add(fromFileStatus(rootStatus)); } final FileUpdateKey updateKey = new FileUpdateKey().setCachedEntitiesList(cachedEntities); final boolean hasDirectories = fileSelection.containsDirectories(); // Expand selection by copying it first used to check extensions of files in directory. final FileSelection fileSelectionWithoutDir = hasDirectories? fileSelection.minusDirectories(): fileSelection; if(fileSelectionWithoutDir == null){ // no files in the found directory, not a table. return null; } return formatPlugin.getDatasetAccessor(oldConfig, fs, fileSelectionWithoutDir, this, datasetPath, datasetPath.getName(), updateKey); }
final List<FileSystemCachedEntity> cachedEntities = fileUpdateKey.getCachedEntitiesList(); for (int i = 0; i < cachedEntities.size(); ++i) { final FileSystemCachedEntity cachedEntity = cachedEntities.get(i);
@Override public CheckResult checkReadSignature(ByteString key, final DatasetConfig oldConfig, DatasetRetrievalOptions retrievalOptions) throws Exception { final FileUpdateKey fileUpdateKey = new FileUpdateKey(); ProtostuffIOUtil.mergeFrom(key.toByteArray(), fileUpdateKey, FileUpdateKey.getSchema()); fileUpdateKey.getCachedEntitiesList() == null || fileUpdateKey.getCachedEntitiesList().isEmpty()) {
final FileUpdateKey updateKey = new FileUpdateKey().setCachedEntitiesList(cachedEntities);