public FileConfig newMessage() { return new FileConfig(); }
private static FileFormat asFormat(NamespaceKey key, Path path, boolean isFolder) { String name = path.getName(); if(name.endsWith(".zip")) { name = name.substring(0, name.length() - 4); } if(name.endsWith(".gz")) { name = name.substring(0, name.length() - 3); } final FileConfig config = new FileConfig() .setCtime(System.currentTimeMillis()) .setFullPathList(key.getPathComponents()) .setName(key.getName()) .setType(FileFormat.getFileFormatType(Collections.singletonList(FilenameUtils.getExtension(name)))) .setTag(null); return isFolder ? FileFormat.getForFolder(config) : FileFormat.getForFile(config); }
@Deprecated public FileFormat getDefaultFileFormat(SourceName sourceName, SourceFilePath sourceFilePath) { final FileConfig config = new FileConfig(); config.setCtime(System.currentTimeMillis()); config.setFullPathList(sourceFilePath.toPathList()); config.setName(sourceFilePath.getFileName().getName()); config.setType(FileFormat.getFileFormatType(singletonList(FilenameUtils.getExtension(config.getName())))); config.setTag(null); return FileFormat.getForFile(config); }
protected FileFormat getUnknownFileFormat(SourceName sourceName, SourceFilePath sourceFilePath) { final FileConfig config = new FileConfig(); config.setCtime(System.currentTimeMillis()); config.setFullPathList(sourceFilePath.toPathList()); config.setName(sourceFilePath.getFileName().getName()); config.setType(FileType.UNKNOWN); config.setTag(null); return FileFormat.getForFile(config); }
/** * Get default file format for a directory in filesystem * @param sourceName name of source * @param sourceFolderPath path to directory * @param user user name * @return {@code FileFormat} format settings * @throws IOException on filesystem related errors * @throws NamespaceException on invalid namespace operation * @throws PhysicalDatasetNotFoundException if file/folder is marked as physical dataset but is missing from namespace */ @Deprecated public FileFormat getDefaultFileFormat(SourceName sourceName, SourceFolderPath sourceFolderPath, String user) throws IOException, NamespaceException, PhysicalDatasetNotFoundException { final FileConfig config = new FileConfig(); config.setCtime(System.currentTimeMillis()); config.setFullPathList(sourceFolderPath.toPathList()); config.setName(sourceFolderPath.getFolderName().getName()); NamespaceTree ns = listFolder(sourceName, sourceFolderPath, user); if (!ns.getFiles().isEmpty()) { config.setType(FileFormat.getFileFormatType(singletonList(FilenameUtils.getExtension(ns.getFiles().get(0).getName())))); } else { config.setType(FileType.UNKNOWN); } config.setTag(null); return FileFormat.getForFolder(config); }
final FileConfig config = new FileConfig(); try {
public static void addFile(NamespaceService ns, List<String> path) throws Exception { NamespaceKey filePath = new NamespaceKey(path); final boolean isHome = path.get(0).startsWith("@"); final DatasetConfig ds = new DatasetConfig() .setType(isHome ? DatasetType.PHYSICAL_DATASET_HOME_FILE : DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig())); ns.addOrUpdateDataset(filePath, ds); }
@JsonIgnore @SuppressWarnings({ "rawtypes", "unchecked" }) public FileConfig asFileConfig() { buffer.clear(); FileConfig fc = new FileConfig(); fc.setType(getFileType()); fc.setName(name); fc.setOwner(owner); fc.setCtime(ctime); fc.setType(getFileType()); fc.setTag(getVersion()); fc.setLocation(location); byte[] bytes = ProtobufIOUtil.toByteArray(this, (Schema) getPrivateSchema(), buffer); fc.setExtendedConfig(ByteString.copyFrom(bytes)); fc.setFullPathList(fullPath); return fc; }
private static String addJsonTable(String tableName, String... jsonData) throws Exception { final File file = temp.newFile(tableName); final String dataFile = file.getAbsolutePath(); //TODO write each record in a separate file, so we can cause a union type for example try (PrintWriter writer = new PrintWriter(file)) { for (String record : jsonData) { writer.println(record); } } final DatasetPath path = new DatasetPath(ImmutableList.of("dfs", dataFile)); final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return dataFile; } }
public void addJson(DatasetPath path, DatasetPath vdsPath) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); createDatasetFromParentAndSave(vdsPath, path.toPathString()); }
protected static DatasetConfig addJson(DatasetPath path) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return nsService.getDataset(path.toNamespaceKey()); }
protected DatasetConfig addDataSet(DatasetPath path) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FOLDER) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.PARQUET).setCtime(1L).setOwner (DEFAULT_USERNAME)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return nsService.getDataset(path.toNamespaceKey()); } }
.buildPost(Entity.json(new FileConfig())), File.class); FileFormat file3Format = file3.getFileFormat().getFileFormat();