@Override public FileConfig getFormatSettings() { return config.getPhysicalDataset().getFormatSettings(); }
@Override public SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig oldConfig, DatasetRetrievalOptions retrievalOptions) throws Exception { FormatPluginConfig formatPluginConfig = null; PhysicalDataset physicalDataset = oldConfig == null ? null : oldConfig.getPhysicalDataset(); if(physicalDataset != null && physicalDataset.getFormatSettings() != null){ formatPluginConfig = PhysicalDatasetUtils.toFormatPlugin(physicalDataset.getFormatSettings(), Collections.<String>emptyList()); } return getDatasetWithFormat(datasetPath, oldConfig, formatPluginConfig, retrievalOptions, null); }
@Override public FileConfig getFormatSettings() { loadIfNecessary(); return datasetConfig.getPhysicalDataset().getFormatSettings(); }
@Override public SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig oldConfig, DatasetRetrievalOptions retrievalOptions) throws Exception { if (datasetPath.size() <= 1) { return null; } PhysicalDataset physicalDataset = oldConfig == null ? null : oldConfig.getPhysicalDataset(); FormatPluginConfig pluginConfig = null; try { final FileSystemWrapper fs = getFs(); final FileConfig fileConfig; if(physicalDataset != null && physicalDataset.getFormatSettings() != null){ fileConfig = physicalDataset.getFormatSettings(); }else{ final DatasetConfig datasetConfig =getContext().getNamespaceService(SystemUser.SYSTEM_USERNAME).getDataset(datasetPath); if (!(datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FILE || datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FOLDER)) { throw new IllegalArgumentException(format("Table %s does not belong to home space", datasetPath.toString())); } fileConfig = datasetConfig.getPhysicalDataset().getFormatSettings(); } pluginConfig = PhysicalDatasetUtils.toFormatPlugin(fileConfig, Collections.<String>emptyList()); final FormatPlugin formatPlugin = formatCreator.newFormatPlugin(pluginConfig); return getDataset(datasetPath, oldConfig, formatPlugin, fs, fileConfig); } catch (NamespaceNotFoundException nfe){ FormatPluginConfig formatPluginConfig = null; if(physicalDataset != null && physicalDataset.getFormatSettings() != null){ formatPluginConfig = PhysicalDatasetUtils.toFormatPlugin(physicalDataset.getFormatSettings(), Collections.<String>emptyList()); } return getDatasetWithFormat(datasetPath, oldConfig, formatPluginConfig, retrievalOptions, null); } }
private Collection<FsPermissionTask> getSplitPermissiomTasks(DatasetConfig datasetConfig, FileSystemWrapper userFs, String user) { final SplitsPointer splitsPointer = DatasetSplitsPointer.of(context.getNamespaceService(user), datasetConfig); final boolean isParquet = datasetConfig.getPhysicalDataset().getFormatSettings().getType() == FileType.PARQUET; final List<FsPermissionTask> fsPermissionTasks = Lists.newArrayList(); final List<Path> batch = Lists.newArrayList(); for (DatasetSplit split: splitsPointer.getSplitIterable()) { final Path filePath; if (isParquet) { filePath = new Path(PARQUET_DATASET_SPLIT_XATTR_SERIALIZER.revert(split.getExtendedProperty().toByteArray()).getPath()); } else { filePath = new Path(EASY_DATASET_SPLIT_XATTR_SERIALIZER.revert(split.getExtendedProperty().toByteArray()).getPath()); } batch.add(filePath); if (batch.size() == PERMISSION_CHECK_TASK_BATCH_SIZE) { // make a copy of batch fsPermissionTasks.add(new FsPermissionTask(userFs, new ArrayList<>(batch), FsAction.READ)); batch.clear(); } } if (!batch.isEmpty()) { fsPermissionTasks.add(new FsPermissionTask(userFs, batch, FsAction.READ)); } return fsPermissionTasks; }
/** * Check with namespace format setting on a file/directory. * @param namespaceService PhysicalDatasetService namespaceService * @param schemaPath parent path * @param tableName table name (file/directory name) * @param fileSelection file/files selected under directory. * @return {@code FormatPluginConfig} that should be used for creating Dremio table. */ public static FormatPluginConfig getPhysicalDatasetProperties(NamespaceService namespaceService, final List<String> schemaPath, final String tableName, final FileSelection fileSelection) { // If the execution engine is being tested outside the context of a Dremio instance, there won't be a dataset service. if(namespaceService == null) { return null; } final List<String> tableSchemaPath = new ArrayList<>(schemaPath); tableSchemaPath.addAll(PathUtils.toPathComponents(new Path(tableName))); try { final DatasetConfig config = namespaceService.getDataset(new NamespaceKey(tableSchemaPath)); return toFormatPlugin(config.getPhysicalDataset().getFormatSettings(), fileSelection.getExtensions()); } catch (NamespaceException e) { logger.debug("Failed to get physical dataset properties for table {} error {}", PathUtils.constructFullPath(tableSchemaPath), e); } return null; }
public void deleteHomeDataset(DatasetConfig config, String version) throws IOException, NamespaceException { FileConfig formatSettings = config.getPhysicalDataset().getFormatSettings(); homeFileTool.deleteFile(formatSettings.getLocation()); namespaceService.deleteDataset(new NamespaceKey(config.getFullPathList()), version); }
if (oldDatasetConfig != null && oldDatasetConfig.getPhysicalDataset().getFormatSettings().equals(datasetConfig.getPhysicalDataset().getFormatSettings())) { return userNamespaceService.tryCreatePhysicalDataset(datasetPath, datasetConfig, attributes); new NamespaceKey(ImmutableList.<String>builder() .add("__home") // TODO (AH) hack. .addAll(PathUtils.toPathComponents(datasetConfig.getPhysicalDataset().getFormatSettings().getLocation())).build()), datasetConfig, false); if (datasetAccessor == null) {
Objects.equals(currentConfig.getPhysicalDataset().getFormatSettings(), datasetConfig.getPhysicalDataset().getFormatSettings())) && Objects.equals(DatasetHelper.getSchemaBytes(currentConfig), DatasetHelper.getSchemaBytes(datasetConfig))) { return false;
@Override protected SourceTableDefinition getDatasetWithFormat(NamespaceKey datasetPath, DatasetConfig oldConfig, FormatPluginConfig formatPluginConfig, DatasetRetrievalOptions retrievalOptions, String user) throws Exception { try{ final FileSystemWrapper fs = getFs(); final DatasetConfig datasetConfig = getContext().getNamespaceService(SystemUser.SYSTEM_USERNAME).getDataset(datasetPath); if (!(datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FILE || datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FOLDER)) { throw new IllegalArgumentException(format("Table %s does not belong to home space", datasetPath.toString())); } final FormatPlugin formatPlugin = formatCreator.newFormatPlugin(formatPluginConfig); return getDataset(datasetPath, oldConfig, formatPlugin, fs, datasetConfig.getPhysicalDataset().getFormatSettings()); } catch (NamespaceNotFoundException nfe){ if(formatPluginConfig == null) { // a home file can only be read from the namespace or using a format options. Without either, it is invalid, return nothing. return null; } return super.getDatasetWithFormat(new NamespaceKey(relativePath(datasetPath.getPathComponents(), getConfig().getPath())), oldConfig, formatPluginConfig, retrievalOptions, null); } }
public static PhysicalDatasetConfig toPhysicalDatasetConfig(DatasetConfig datasetConfig) { checkNotNull(datasetConfig.getPhysicalDataset()); final com.dremio.service.namespace.dataset.proto.PhysicalDataset physicalDataset = datasetConfig.getPhysicalDataset(); final PhysicalDatasetConfig physicalDatasetConfig = new PhysicalDatasetConfig(); physicalDatasetConfig.setFormatSettings(physicalDataset.getFormatSettings()); physicalDatasetConfig.setFullPathList(datasetConfig.getFullPathList()); physicalDatasetConfig.setType(datasetConfig.getType()); physicalDatasetConfig.setName(datasetConfig.getName()); physicalDatasetConfig.setTag(datasetConfig.getTag()); physicalDatasetConfig.setId(datasetConfig.getId().getId()); return physicalDatasetConfig; }
public static FileConfig toFileConfig(DatasetConfig datasetConfig) { final FileConfig fileConfig = datasetConfig.getPhysicalDataset().getFormatSettings(); fileConfig.setCtime(datasetConfig.getCreatedAt()); fileConfig.setTag(datasetConfig.getTag()); fileConfig.setOwner(datasetConfig.getOwner()); fileConfig.setFullPathList(datasetConfig.getFullPathList()); fileConfig.setName(datasetConfig.getName()); return fileConfig; }
FileConfig formatSettings = physicalDataset.getFormatSettings();