public static void addFile(NamespaceService ns, List<String> path) throws Exception { NamespaceKey filePath = new NamespaceKey(path); final boolean isHome = path.get(0).startsWith("@"); final DatasetConfig ds = new DatasetConfig() .setType(isHome ? DatasetType.PHYSICAL_DATASET_HOME_FILE : DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig())); ns.addOrUpdateDataset(filePath, ds); }
@Override public FileConfig getFormatSettings() { return config.getPhysicalDataset().getFormatSettings(); }
public boolean isApproximateStatsAllowed() { PhysicalDataset pd = dataset.getDatasetConfig().getPhysicalDataset(); if(pd == null) { return false; } return pd.getAllowApproxStats() == null ? false : pd.getAllowApproxStats(); }
public PhysicalDataset newMessage() { return new PhysicalDataset(); }
currentDatasetConfig.getPhysicalDataset().setAllowApproxStats(dataset.getApproximateStatisticsAllowed()); currentDatasetConfig.getPhysicalDataset().setFormatSettings(dataset.getFormat().asFileConfig());
FileConfig formatSettings = physicalDataset.getFormatSettings(); null, format, physicalDataset.getAllowApproxStats() );
config.getPhysicalDataset().setAllowApproxStats(sqlSetApprox.isEnable()); namespaceService.addOrUpdateDataset(path, config); return singletonList(successful(String.format("Successfully updated table '%s' from namespace.", table.getPath())));
private DatasetBuilder(HiveClient client, String user, NamespaceKey datasetPath, boolean ignoreAuthzErrors, StatsEstimationParameters statsParams, HiveConf hiveConf, String dbName, String tableName, Table table, DatasetConfig oldConfig){ if(oldConfig == null){ datasetConfig = new DatasetConfig() .setPhysicalDataset(new PhysicalDataset()) .setId(new EntityId().setId(UUID.randomUUID().toString())); } else { datasetConfig = oldConfig; // We're rewriting the read definition. Delete the old one. oldConfig.setReadDefinition(null); } this.client = client; this.user = user; this.datasetPath = datasetPath; this.hiveConf = hiveConf; this.table = table; this.dbName = dbName; this.tableName = tableName; this.ignoreAuthzErrors = ignoreAuthzErrors; this.statsParams = statsParams; }
public static DatasetConfig toDatasetConfig(PhysicalDatasetConfig physicalDatasetConfig, String owner) { final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setOwner(owner); datasetConfig.setFullPathList(physicalDatasetConfig.getFullPathList()); datasetConfig.setName(physicalDatasetConfig.getName()); datasetConfig.setType(physicalDatasetConfig.getType()); datasetConfig.setTag(physicalDatasetConfig.getTag()); datasetConfig.setPhysicalDataset(new com.dremio.service.namespace.dataset.proto.PhysicalDataset().setFormatSettings( physicalDatasetConfig.getFormatSettings())); return datasetConfig; }
createSourceFolders(datasetPath); if(dataset.getPhysicalDataset() == null){ dataset.setPhysicalDataset(new PhysicalDataset()); case PHYSICAL_DATASET_HOME_FOLDER: { if(dataset.getPhysicalDataset() == null){ dataset.setPhysicalDataset(new PhysicalDataset());
@Override public SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig oldConfig, DatasetRetrievalOptions retrievalOptions) throws Exception { FormatPluginConfig formatPluginConfig = null; PhysicalDataset physicalDataset = oldConfig == null ? null : oldConfig.getPhysicalDataset(); if(physicalDataset != null && physicalDataset.getFormatSettings() != null){ formatPluginConfig = PhysicalDatasetUtils.toFormatPlugin(physicalDataset.getFormatSettings(), Collections.<String>emptyList()); } return getDatasetWithFormat(datasetPath, oldConfig, formatPluginConfig, retrievalOptions, null); }
private UserBitShared.DatasetProfile buildDatasetProfile(final DremioTable t) { try { if (t instanceof ViewTable) { final ViewTable view = (ViewTable) t; return UserBitShared.DatasetProfile.newBuilder() .setDatasetPath(t.getPath().getSchemaPath()) .setType(UserBitShared.DatasetType.VDS) .setSql(view.getView().getSql()) .build(); } else { Boolean allowApproxStats = false; PhysicalDataset physicalDataset = t.getDatasetConfig().getPhysicalDataset(); if (physicalDataset != null) { allowApproxStats = physicalDataset.getAllowApproxStats(); } return UserBitShared.DatasetProfile.newBuilder() .setDatasetPath(t.getPath().getSchemaPath()) .setType(UserBitShared.DatasetType.PDS) .setBatchSchema(ByteString.copyFrom(t.getSchema().serialize())) .setAllowApproxStats(allowApproxStats) .build(); } } catch (Exception e) { logger.warn("Couldn't build dataset profile for table {}", t.getPath().getSchemaPath(), e); } return null; }
.setSchemaVersion(DatasetHelper.CURRENT_VERSION) .setRecordSchema(schema.toByteString()) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(toFileFormat(formatPlugin).asFileConfig().setLocation(selection.getSelectionRoot())));
}else { dataset = new DatasetConfig() .setPhysicalDataset(new PhysicalDataset()) .setId(new EntityId().setId(UUID.randomUUID().toString())) .setFullPathList(name.getPathComponents())
@Override public FileConfig getFormatSettings() { loadIfNecessary(); return datasetConfig.getPhysicalDataset().getFormatSettings(); }
public static DatasetConfig toDatasetConfig(FileConfig fileConfig, DatasetType datasetType, String owner, EntityId id) { final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setOwner(owner); datasetConfig.setFullPathList(fileConfig.getFullPathList()); datasetConfig.setName(fileConfig.getName()); datasetConfig.setOwner(fileConfig.getOwner()); datasetConfig.setTag(fileConfig.getTag()); datasetConfig.setType(datasetType); datasetConfig.setCreatedAt(fileConfig.getCtime()); datasetConfig.setId(id); datasetConfig.setPhysicalDataset(new com.dremio.service.namespace.dataset.proto.PhysicalDataset().setFormatSettings(fileConfig)); return datasetConfig; }
@Override public DatasetConfig getDataset() throws Exception { final DatasetConfig dataset; if(oldDataset == null) { dataset = new DatasetConfig() .setFullPathList(key.getPathComponents()) .setId(new EntityId(UUID.randomUUID().toString())) .setType(DatasetType.PHYSICAL_DATASET); } else { dataset = oldDataset; } return dataset .setName(key.getName()) .setReadDefinition(new ReadDefinition() .setScanStats(new ScanStats().setRecordCount(100l) .setScanFactor(ScanCostFactor.OTHER.getFactor()))) .setOwner(SystemUser.SYSTEM_USERNAME) .setPhysicalDataset(new PhysicalDataset()) .setRecordSchema(getSchema().toByteString()) .setSchemaVersion(DatasetHelper.CURRENT_VERSION); }
@Override public SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig oldConfig, DatasetRetrievalOptions retrievalOptions) throws Exception { if (datasetPath.size() <= 1) { return null; } PhysicalDataset physicalDataset = oldConfig == null ? null : oldConfig.getPhysicalDataset(); FormatPluginConfig pluginConfig = null; try { final FileSystemWrapper fs = getFs(); final FileConfig fileConfig; if(physicalDataset != null && physicalDataset.getFormatSettings() != null){ fileConfig = physicalDataset.getFormatSettings(); }else{ final DatasetConfig datasetConfig =getContext().getNamespaceService(SystemUser.SYSTEM_USERNAME).getDataset(datasetPath); if (!(datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FILE || datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FOLDER)) { throw new IllegalArgumentException(format("Table %s does not belong to home space", datasetPath.toString())); } fileConfig = datasetConfig.getPhysicalDataset().getFormatSettings(); } pluginConfig = PhysicalDatasetUtils.toFormatPlugin(fileConfig, Collections.<String>emptyList()); final FormatPlugin formatPlugin = formatCreator.newFormatPlugin(pluginConfig); return getDataset(datasetPath, oldConfig, formatPlugin, fs, fileConfig); } catch (NamespaceNotFoundException nfe){ FormatPluginConfig formatPluginConfig = null; if(physicalDataset != null && physicalDataset.getFormatSettings() != null){ formatPluginConfig = PhysicalDatasetUtils.toFormatPlugin(physicalDataset.getFormatSettings(), Collections.<String>emptyList()); } return getDatasetWithFormat(datasetPath, oldConfig, formatPluginConfig, retrievalOptions, null); } }
dacSample1.setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE); dacSample1.setName(allTypesJson); dacSample1.setPhysicalDataset(new PhysicalDataset().setFormatSettings(new JsonFileConfig().asFileConfig())); namespaceService.addOrUpdateDataset(dacSampleAllTypes.toNamespaceKey(), dacSample1); dacSample1.setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE); dacSample1.setName("dac-sample1.json"); dacSample1.setPhysicalDataset(new PhysicalDataset().setFormatSettings(new JsonFileConfig().asFileConfig())); namespaceService.addOrUpdateDataset(dacSample1Path.toNamespaceKey(), dacSample1); dacSample2.setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE); dacSample2.setName("dac-sample2.json"); dacSample2.setPhysicalDataset(new PhysicalDataset().setFormatSettings(new JsonFileConfig().asFileConfig())); namespaceService.addOrUpdateDataset(dacSample2Path.toNamespaceKey(), dacSample2);
@Override public DatasetConfig getDataset() { final DatasetConfig dataset; if(oldDataset == null) { dataset = new DatasetConfig() .setFullPathList(key.getPathComponents()) .setId(new EntityId(UUID.randomUUID().toString())) .setType(DatasetType.PHYSICAL_DATASET); } else { dataset = oldDataset; } return dataset .setName(key.getName()) .setReadDefinition(new ReadDefinition() .setScanStats(new ScanStats().setRecordCount(100l) .setScanFactor(ScanCostFactor.OTHER.getFactor()))) .setOwner(SystemUser.SYSTEM_USERNAME) .setPhysicalDataset(new PhysicalDataset()) .setRecordSchema(getSchema().toByteString()) .setSchemaVersion(DatasetHelper.CURRENT_VERSION); }