private static FileFormat asFormat(NamespaceKey key, Path path, boolean isFolder) { String name = path.getName(); if(name.endsWith(".zip")) { name = name.substring(0, name.length() - 4); } if(name.endsWith(".gz")) { name = name.substring(0, name.length() - 3); } final FileConfig config = new FileConfig() .setCtime(System.currentTimeMillis()) .setFullPathList(key.getPathComponents()) .setName(key.getName()) .setType(FileFormat.getFileFormatType(Collections.singletonList(FilenameUtils.getExtension(name)))) .setTag(null); return isFolder ? FileFormat.getForFolder(config) : FileFormat.getForFile(config); }
@Deprecated public FileFormat getDefaultFileFormat(SourceName sourceName, SourceFilePath sourceFilePath) { final FileConfig config = new FileConfig(); config.setCtime(System.currentTimeMillis()); config.setFullPathList(sourceFilePath.toPathList()); config.setName(sourceFilePath.getFileName().getName()); config.setType(FileFormat.getFileFormatType(singletonList(FilenameUtils.getExtension(config.getName())))); config.setTag(null); return FileFormat.getForFile(config); }
protected FileFormat getUnknownFileFormat(SourceName sourceName, SourceFilePath sourceFilePath) { final FileConfig config = new FileConfig(); config.setCtime(System.currentTimeMillis()); config.setFullPathList(sourceFilePath.toPathList()); config.setName(sourceFilePath.getFileName().getName()); config.setType(FileType.UNKNOWN); config.setTag(null); return FileFormat.getForFile(config); }
/** * Get default file format for a directory in filesystem * @param sourceName name of source * @param sourceFolderPath path to directory * @param user user name * @return {@code FileFormat} format settings * @throws IOException on filesystem related errors * @throws NamespaceException on invalid namespace operation * @throws PhysicalDatasetNotFoundException if file/folder is marked as physical dataset but is missing from namespace */ @Deprecated public FileFormat getDefaultFileFormat(SourceName sourceName, SourceFolderPath sourceFolderPath, String user) throws IOException, NamespaceException, PhysicalDatasetNotFoundException { final FileConfig config = new FileConfig(); config.setCtime(System.currentTimeMillis()); config.setFullPathList(sourceFolderPath.toPathList()); config.setName(sourceFolderPath.getFolderName().getName()); NamespaceTree ns = listFolder(sourceName, sourceFolderPath, user); if (!ns.getFiles().isEmpty()) { config.setType(FileFormat.getFileFormatType(singletonList(FilenameUtils.getExtension(ns.getFiles().get(0).getName())))); } else { config.setType(FileType.UNKNOWN); } config.setTag(null); return FileFormat.getForFolder(config); }
public File getFileDataset(SourceName source, final SourceFilePath filePath, String owner) throws PhysicalDatasetNotFoundException, NamespaceException { final PhysicalDatasetConfig physicalDatasetConfig = getFilesystemPhysicalDataset(filePath, DatasetType.PHYSICAL_DATASET_SOURCE_FILE); final FileConfig fileConfig = physicalDatasetConfig.getFormatSettings(); fileConfig.setOwner(owner); fileConfig.setTag(physicalDatasetConfig.getTag()); final File file = File.newInstance(physicalDatasetConfig.getId(), filePath, FileFormat.getForFile(fileConfig), datasetService.getJobsCount(filePath.toNamespaceKey()), false, false, fileConfig.getType() != FileType.UNKNOWN, null ); return file; }
public static FileConfig toFileConfig(DatasetConfig datasetConfig) { final FileConfig fileConfig = datasetConfig.getPhysicalDataset().getFormatSettings(); fileConfig.setCtime(datasetConfig.getCreatedAt()); fileConfig.setTag(datasetConfig.getTag()); fileConfig.setOwner(datasetConfig.getOwner()); fileConfig.setFullPathList(datasetConfig.getFullPathList()); fileConfig.setName(datasetConfig.getName()); return fileConfig; }
@JsonIgnore @SuppressWarnings({ "rawtypes", "unchecked" }) public FileConfig asFileConfig() { buffer.clear(); FileConfig fc = new FileConfig(); fc.setType(getFileType()); fc.setName(name); fc.setOwner(owner); fc.setCtime(ctime); fc.setType(getFileType()); fc.setTag(getVersion()); fc.setLocation(location); byte[] bytes = ProtobufIOUtil.toByteArray(this, (Schema) getPrivateSchema(), buffer); fc.setExtendedConfig(ByteString.copyFrom(bytes)); fc.setFullPathList(fullPath); return fc; }
fileConfig.setTag(physicalDatasetConfig.getTag());