protected FileFormat getUnknownFileFormat(SourceName sourceName, SourceFilePath sourceFilePath) { final FileConfig config = new FileConfig(); config.setCtime(System.currentTimeMillis()); config.setFullPathList(sourceFilePath.toPathList()); config.setName(sourceFilePath.getFileName().getName()); config.setType(FileType.UNKNOWN); config.setTag(null); return FileFormat.getForFile(config); }
private static FileFormat get(FileConfig fileConfig) { // TODO (Amit H) Remove after defining classes for tsv, csv, and psv FileType fileType = fileConfig.getType(); if (fileType == FileType.CSV || fileType == FileType.TSV || fileType == FileType.PSV) { fileType = FileType.TEXT; } final Class<? extends FileFormat> fileFormatClass = FileFormatDefinitions.CLASS_TYPES.get(fileType); final Schema<FileFormat> schema = (Schema<FileFormat>) FileFormatDefinitions.SCHEMAS.get(fileFormatClass); final FileFormat fileFormat = schema.newMessage(); if (fileConfig.getExtendedConfig() != null) { ProtobufIOUtil.mergeFrom(fileConfig.getExtendedConfig().toByteArray(), fileFormat, schema); } fileFormat.setCtime(fileConfig.getCtime()); fileFormat.setName(fileConfig.getName()); fileFormat.setOwner(fileConfig.getOwner()); fileFormat.setFullPath(fileConfig.getFullPathList()); fileFormat.setVersion(fileConfig.getTag()); fileFormat.setLocation(fileConfig.getLocation()); return fileFormat; }
@PUT @Path("file_format/{path: .*}") @Produces(MediaType.APPLICATION_JSON) @Consumes(MediaType.APPLICATION_JSON) public FileFormatUI saveFormatSettings(FileFormat fileFormat, @PathParam("path") String path) throws FileNotFoundException, HomeNotFoundException, NamespaceException { FilePath filePath = FilePath.fromURLPath(homeName, path); // merge file configs final DatasetConfig existingDSConfig = namespaceService.getDataset(filePath.toNamespaceKey()); final FileConfig oldConfig = toFileConfig(existingDSConfig); final FileConfig newConfig = fileFormat.asFileConfig(); newConfig.setCtime(oldConfig.getCtime()); newConfig.setFullPathList(oldConfig.getFullPathList()); newConfig.setName(oldConfig.getName()); newConfig.setOwner(oldConfig.getOwner()); newConfig.setLocation(oldConfig.getLocation()); catalog.createOrUpdateDataset(namespaceService, new NamespaceKey(HomeFileSystemStoragePlugin.HOME_PLUGIN_NAME), filePath.toNamespaceKey(), toDatasetConfig(newConfig, DatasetType.PHYSICAL_DATASET_HOME_FILE, securityContext.getUserPrincipal().getName(), existingDSConfig.getId())); return new FileFormatUI(FileFormat.getForFile(newConfig), filePath); }
@JsonIgnore @SuppressWarnings({ "rawtypes", "unchecked" }) public FileConfig asFileConfig() { buffer.clear(); FileConfig fc = new FileConfig(); fc.setType(getFileType()); fc.setName(name); fc.setOwner(owner); fc.setCtime(ctime); fc.setType(getFileType()); fc.setTag(getVersion()); fc.setLocation(location); byte[] bytes = ProtobufIOUtil.toByteArray(this, (Schema) getPrivateSchema(), buffer); fc.setExtendedConfig(ByteString.copyFrom(bytes)); fc.setFullPathList(fullPath); return fc; }
public static FileConfig toFileConfig(DatasetConfig datasetConfig) { final FileConfig fileConfig = datasetConfig.getPhysicalDataset().getFormatSettings(); fileConfig.setCtime(datasetConfig.getCreatedAt()); fileConfig.setTag(datasetConfig.getTag()); fileConfig.setOwner(datasetConfig.getOwner()); fileConfig.setFullPathList(datasetConfig.getFullPathList()); fileConfig.setName(datasetConfig.getName()); return fileConfig; }
final FileConfig config = new FileConfig(); try { config.setLocation(stagingLocation.toString()); config.setName(filePath.getLeaf().getName()); config.setCtime(System.currentTimeMillis()); config.setFullPathList(filePath.toPathList()); config.setOwner(securityContext.getUserPrincipal().getName()); config.setType(FileFormat.getFileFormatType(Collections.singletonList(extension))); } catch (IOException ioe) { throw new DACException("Error writing to file at " + filePath, ioe);
protected DatasetConfig addDataSet(DatasetPath path) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FOLDER) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.PARQUET).setCtime(1L).setOwner (DEFAULT_USERNAME)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return nsService.getDataset(path.toNamespaceKey()); } }
public static DatasetConfig toDatasetConfig(FileConfig fileConfig, DatasetType datasetType, String owner, EntityId id) { final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setOwner(owner); datasetConfig.setFullPathList(fileConfig.getFullPathList()); datasetConfig.setName(fileConfig.getName()); datasetConfig.setOwner(fileConfig.getOwner()); datasetConfig.setTag(fileConfig.getTag()); datasetConfig.setType(datasetType); datasetConfig.setCreatedAt(fileConfig.getCtime()); datasetConfig.setId(id); datasetConfig.setPhysicalDataset(new com.dremio.service.namespace.dataset.proto.PhysicalDataset().setFormatSettings(fileConfig)); return datasetConfig; }
public File getFileDataset(SourceName source, final SourceFilePath filePath, String owner) throws PhysicalDatasetNotFoundException, NamespaceException { final PhysicalDatasetConfig physicalDatasetConfig = getFilesystemPhysicalDataset(filePath, DatasetType.PHYSICAL_DATASET_SOURCE_FILE); final FileConfig fileConfig = physicalDatasetConfig.getFormatSettings(); fileConfig.setOwner(owner); fileConfig.setTag(physicalDatasetConfig.getTag()); final File file = File.newInstance(physicalDatasetConfig.getId(), filePath, FileFormat.getForFile(fileConfig), datasetService.getJobsCount(filePath.toNamespaceKey()), false, false, fileConfig.getType() != FileType.UNKNOWN, null ); return file; }
private static String addJsonTable(String tableName, String... jsonData) throws Exception { final File file = temp.newFile(tableName); final String dataFile = file.getAbsolutePath(); //TODO write each record in a separate file, so we can cause a union type for example try (PrintWriter writer = new PrintWriter(file)) { for (String record : jsonData) { writer.println(record); } } final DatasetPath path = new DatasetPath(ImmutableList.of("dfs", dataFile)); final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return dataFile; } }
@Override public int getOperatorType() { return EasyGroupScan.getEasyScanOperatorType(fileConfig.getType()); } }
assertEquals(fileConfig.getFieldDelimiter(), format1.getFieldDelimiter()); assertEquals(fileConfig.getExtractHeader(), format1.getExtractHeader()); assertEquals(fileConfig.asFileConfig().getType(), format1.asFileConfig().getType()); assertEquals(fileConfig.asFileConfig().getOwner(), format1.asFileConfig().getOwner());
public FileConfig newMessage() { return new FileConfig(); }
if (oldDatasetConfig != null && oldDatasetConfig.getPhysicalDataset().getFormatSettings().equals(datasetConfig.getPhysicalDataset().getFormatSettings())) { return userNamespaceService.tryCreatePhysicalDataset(datasetPath, datasetConfig, attributes); new NamespaceKey(ImmutableList.<String>builder() .add("__home") // TODO (AH) hack. .addAll(PathUtils.toPathComponents(datasetConfig.getPhysicalDataset().getFormatSettings().getLocation())).build()), datasetConfig, false); if (datasetAccessor == null) {
private SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig oldConfig, FormatPlugin formatPlugin, FileSystemWrapper fs, FileConfig fileConfig) throws IOException { final List<FileSystemCachedEntity> cachedEntities = Lists.newArrayList(); final FileStatus rootStatus = fs.getFileStatus(new Path(fileConfig.getLocation())); final Path combined = new Path("/", PathUtils.removeLeadingSlash(fileConfig.getLocation())); final FileSelection fileSelection = FileSelection.create(fs, combined); if (fileSelection == null) { return null; } // first entity is always a root if (rootStatus.isDirectory()) { cachedEntities.add(fromFileStatus(rootStatus)); } for (FileStatus dirStatus: fileSelection.getAllDirectories()) { cachedEntities.add(fromFileStatus(dirStatus)); } if(cachedEntities.isEmpty()){ // this is a single file. cachedEntities.add(fromFileStatus(rootStatus)); } final FileUpdateKey updateKey = new FileUpdateKey().setCachedEntitiesList(cachedEntities); final boolean hasDirectories = fileSelection.containsDirectories(); // Expand selection by copying it first used to check extensions of files in directory. final FileSelection fileSelectionWithoutDir = hasDirectories? fileSelection.minusDirectories(): fileSelection; if(fileSelectionWithoutDir == null){ // no files in the found directory, not a table. return null; } return formatPlugin.getDatasetAccessor(oldConfig, fs, fileSelectionWithoutDir, this, datasetPath, datasetPath.getName(), updateKey); }
.setRecordSchema(schema.toByteString()) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(toFileFormat(formatPlugin).asFileConfig().setLocation(selection.getSelectionRoot())));
protected static DatasetConfig addJson(DatasetPath path) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return nsService.getDataset(path.toNamespaceKey()); }
private static boolean isParquetDataset(TableMetadata datasetPointer) { return datasetPointer.getFormatSettings().getType() == FileType.PARQUET; } }
public static void addFile(NamespaceService ns, List<String> path) throws Exception { NamespaceKey filePath = new NamespaceKey(path); final boolean isHome = path.get(0).startsWith("@"); final DatasetConfig ds = new DatasetConfig() .setType(isHome ? DatasetType.PHYSICAL_DATASET_HOME_FILE : DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig())); ns.addOrUpdateDataset(filePath, ds); }