private SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig oldConfig, FormatPlugin formatPlugin, FileSystemWrapper fs, FileConfig fileConfig) throws IOException { final List<FileSystemCachedEntity> cachedEntities = Lists.newArrayList(); final FileStatus rootStatus = fs.getFileStatus(new Path(fileConfig.getLocation())); final Path combined = new Path("/", PathUtils.removeLeadingSlash(fileConfig.getLocation())); final FileSelection fileSelection = FileSelection.create(fs, combined); if (fileSelection == null) { return null; } // first entity is always a root if (rootStatus.isDirectory()) { cachedEntities.add(fromFileStatus(rootStatus)); } for (FileStatus dirStatus: fileSelection.getAllDirectories()) { cachedEntities.add(fromFileStatus(dirStatus)); } if(cachedEntities.isEmpty()){ // this is a single file. cachedEntities.add(fromFileStatus(rootStatus)); } final FileUpdateKey updateKey = new FileUpdateKey().setCachedEntitiesList(cachedEntities); final boolean hasDirectories = fileSelection.containsDirectories(); // Expand selection by copying it first used to check extensions of files in directory. final FileSelection fileSelectionWithoutDir = hasDirectories? fileSelection.minusDirectories(): fileSelection; if(fileSelectionWithoutDir == null){ // no files in the found directory, not a table. return null; } return formatPlugin.getDatasetAccessor(oldConfig, fs, fileSelectionWithoutDir, this, datasetPath, datasetPath.getName(), updateKey); }
public void deleteHomeDataset(DatasetConfig config, String version) throws IOException, NamespaceException { FileConfig formatSettings = config.getPhysicalDataset().getFormatSettings(); homeFileTool.deleteFile(formatSettings.getLocation()); namespaceService.deleteDataset(new NamespaceKey(config.getFullPathList()), version); }
@PUT @Path("file_format/{path: .*}") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) public FileFormatUI saveFormatSettings(FileFormat fileFormat, @PathParam("path") String path) throws FileNotFoundException, HomeNotFoundException, NamespaceException { FilePath filePath = FilePath.fromURLPath(homeName, path); // merge file configs final DatasetConfig existingDSConfig = namespaceService.getDataset(filePath.toNamespaceKey()); final FileConfig oldConfig = toFileConfig(existingDSConfig); final FileConfig newConfig = fileFormat.asFileConfig(); newConfig.setCtime(oldConfig.getCtime()); newConfig.setFullPathList(oldConfig.getFullPathList()); newConfig.setName(oldConfig.getName()); newConfig.setOwner(oldConfig.getOwner()); newConfig.setLocation(oldConfig.getLocation()); catalog.createOrUpdateDataset(namespaceService, new NamespaceKey(HomeFileSystemStoragePlugin.HOME_PLUGIN_NAME), filePath.toNamespaceKey(), toDatasetConfig(newConfig, DatasetType.PHYSICAL_DATASET_HOME_FILE, securityContext.getUserPrincipal().getName(), existingDSConfig.getId())); return new FileFormatUI(FileFormat.getForFile(newConfig), filePath); }
new NamespaceKey(ImmutableList.<String>builder() .add("__home") // TODO (AH) hack. .addAll(PathUtils.toPathComponents(datasetConfig.getPhysicalDataset().getFormatSettings().getLocation())).build()), datasetConfig, false); if (datasetAccessor == null) {
private static FileFormat get(FileConfig fileConfig) { // TODO (Amit H) Remove after defining classes for tsv, csv, and psv FileType fileType = fileConfig.getType(); if (fileType == FileType.CSV || fileType == FileType.TSV || fileType == FileType.PSV) { fileType = FileType.TEXT; } final Class<? extends FileFormat> fileFormatClass = FileFormatDefinitions.CLASS_TYPES.get(fileType); final Schema<FileFormat> schema = (Schema<FileFormat>) FileFormatDefinitions.SCHEMAS.get(fileFormatClass); final FileFormat fileFormat = schema.newMessage(); if (fileConfig.getExtendedConfig() != null) { ProtobufIOUtil.mergeFrom(fileConfig.getExtendedConfig().toByteArray(), fileFormat, schema); } fileFormat.setCtime(fileConfig.getCtime()); fileFormat.setName(fileConfig.getName()); fileFormat.setOwner(fileConfig.getOwner()); fileFormat.setFullPath(fileConfig.getFullPathList()); fileFormat.setVersion(fileConfig.getTag()); fileFormat.setLocation(fileConfig.getLocation()); return fileFormat; }