public PhysicalDataset newMessage() { return new PhysicalDataset(); }
private DatasetBuilder(HiveClient client, String user, NamespaceKey datasetPath, boolean ignoreAuthzErrors, StatsEstimationParameters statsParams, HiveConf hiveConf, String dbName, String tableName, Table table, DatasetConfig oldConfig){ if(oldConfig == null){ datasetConfig = new DatasetConfig() .setPhysicalDataset(new PhysicalDataset()) .setId(new EntityId().setId(UUID.randomUUID().toString())); } else { datasetConfig = oldConfig; // We're rewriting the read definition. Delete the old one. oldConfig.setReadDefinition(null); } this.client = client; this.user = user; this.datasetPath = datasetPath; this.hiveConf = hiveConf; this.table = table; this.dbName = dbName; this.tableName = tableName; this.ignoreAuthzErrors = ignoreAuthzErrors; this.statsParams = statsParams; }
createSourceFolders(datasetPath); if(dataset.getPhysicalDataset() == null){ dataset.setPhysicalDataset(new PhysicalDataset()); case PHYSICAL_DATASET_HOME_FOLDER: { if(dataset.getPhysicalDataset() == null){ dataset.setPhysicalDataset(new PhysicalDataset());
public static void addFile(NamespaceService ns, List<String> path) throws Exception { NamespaceKey filePath = new NamespaceKey(path); final boolean isHome = path.get(0).startsWith("@"); final DatasetConfig ds = new DatasetConfig() .setType(isHome ? DatasetType.PHYSICAL_DATASET_HOME_FILE : DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig())); ns.addOrUpdateDataset(filePath, ds); }
.setSchemaVersion(DatasetHelper.CURRENT_VERSION) .setRecordSchema(schema.toByteString()) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(toFileFormat(formatPlugin).asFileConfig().setLocation(selection.getSelectionRoot())));
public static DatasetConfig toDatasetConfig(PhysicalDatasetConfig physicalDatasetConfig, String owner) { final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setOwner(owner); datasetConfig.setFullPathList(physicalDatasetConfig.getFullPathList()); datasetConfig.setName(physicalDatasetConfig.getName()); datasetConfig.setType(physicalDatasetConfig.getType()); datasetConfig.setTag(physicalDatasetConfig.getTag()); datasetConfig.setPhysicalDataset(new com.dremio.service.namespace.dataset.proto.PhysicalDataset().setFormatSettings( physicalDatasetConfig.getFormatSettings())); return datasetConfig; }
@Override public DatasetConfig getDataset() throws Exception { final DatasetConfig dataset; if(oldDataset == null) { dataset = new DatasetConfig() .setFullPathList(key.getPathComponents()) .setId(new EntityId(UUID.randomUUID().toString())) .setType(DatasetType.PHYSICAL_DATASET); } else { dataset = oldDataset; } return dataset .setName(key.getName()) .setReadDefinition(new ReadDefinition() .setScanStats(new ScanStats().setRecordCount(100l) .setScanFactor(ScanCostFactor.OTHER.getFactor()))) .setOwner(SystemUser.SYSTEM_USERNAME) .setPhysicalDataset(new PhysicalDataset()) .setRecordSchema(getSchema().toByteString()) .setSchemaVersion(DatasetHelper.CURRENT_VERSION); }
@Override public DatasetConfig getDataset() { final DatasetConfig dataset; if(oldDataset == null) { dataset = new DatasetConfig() .setFullPathList(key.getPathComponents()) .setId(new EntityId(UUID.randomUUID().toString())) .setType(DatasetType.PHYSICAL_DATASET); } else { dataset = oldDataset; } return dataset .setName(key.getName()) .setReadDefinition(new ReadDefinition() .setScanStats(new ScanStats().setRecordCount(100l) .setScanFactor(ScanCostFactor.OTHER.getFactor()))) .setOwner(SystemUser.SYSTEM_USERNAME) .setPhysicalDataset(new PhysicalDataset()) .setRecordSchema(getSchema().toByteString()) .setSchemaVersion(DatasetHelper.CURRENT_VERSION); }
private static SourceTableDefinition newDataset(final String dsPath) { final List<String> path = SqlUtils.parseSchemaPath(dsPath); SourceTableDefinition ret = mock(SourceTableDefinition.class); NamespaceKey datasetName = new NamespaceKey(path); when(ret.getName()).thenReturn(datasetName); BatchSchema schema = BatchSchema.newBuilder() .addField(new Field("string", FieldType.nullable(ArrowType.Utf8.INSTANCE), null)) .build(); DatasetConfig dsConfig = new DatasetConfig() .setName(Util.last(path)) .setFullPathList(path) .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setRecordSchema(ByteString.EMPTY) .setPhysicalDataset( new PhysicalDataset() .setFormatSettings(null)) .setSchemaVersion(DatasetHelper.CURRENT_VERSION) .setRecordSchema(schema.toByteString()) .setReadDefinition(new ReadDefinition()); try { when(ret.getDataset()).thenReturn(dsConfig); } catch (Exception ignored) { } when(ret.getType()).thenReturn(DatasetType.PHYSICAL_DATASET_SOURCE_FILE); when(ret.isSaveable()).thenReturn(true); return ret; }
public static DatasetConfig toDatasetConfig(FileConfig fileConfig, DatasetType datasetType, String owner, EntityId id) { final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setOwner(owner); datasetConfig.setFullPathList(fileConfig.getFullPathList()); datasetConfig.setName(fileConfig.getName()); datasetConfig.setOwner(fileConfig.getOwner()); datasetConfig.setTag(fileConfig.getTag()); datasetConfig.setType(datasetType); datasetConfig.setCreatedAt(fileConfig.getCtime()); datasetConfig.setId(id); datasetConfig.setPhysicalDataset(new com.dremio.service.namespace.dataset.proto.PhysicalDataset().setFormatSettings(fileConfig)); return datasetConfig; }
public void addPhysicalDataset(final DatasetPath path, final DatasetType type) throws Exception { NamespaceKey datasetPath = path.toNamespaceKey(); final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setName(datasetPath.getName()); datasetConfig.setType(type); datasetConfig.setPhysicalDataset(new PhysicalDataset()); getNamespaceService().tryCreatePhysicalDataset(datasetPath, datasetConfig); }
public void addPhysicalDataset(final DatasetPath path, final DatasetType type) throws Exception { NamespaceKey datasetPath = path.toNamespaceKey(); final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setName(datasetPath.getName()); datasetConfig.setType(type); datasetConfig.setPhysicalDataset(new PhysicalDataset()); getNamespaceService().tryCreatePhysicalDataset(datasetPath, datasetConfig); }
private static String addJsonTable(String tableName, String... jsonData) throws Exception { final File file = temp.newFile(tableName); final String dataFile = file.getAbsolutePath(); //TODO write each record in a separate file, so we can cause a union type for example try (PrintWriter writer = new PrintWriter(file)) { for (String record : jsonData) { writer.println(record); } } final DatasetPath path = new DatasetPath(ImmutableList.of("dfs", dataFile)); final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return dataFile; } }
/** * Carry over few properties from old dataset config to new one * @param oldConfig old dataset config from namespace * @param newConfig new dataset config thats about to be saved in namespace */ public static void copyFromOldConfig(DatasetConfig oldConfig, DatasetConfig newConfig) { if(oldConfig == null) { return; } newConfig.setId(oldConfig.getId()); newConfig.setTag(oldConfig.getTag()); newConfig.setCreatedAt(oldConfig.getCreatedAt()); newConfig.setType(oldConfig.getType()); newConfig.setFullPathList(oldConfig.getFullPathList()); newConfig.setOwner(oldConfig.getOwner()); // make sure to copy the acceleration settings from old to new config // newConfig may contain upgrade fileFormat physical settings if (oldConfig.getPhysicalDataset() != null) { if (newConfig.getPhysicalDataset() == null) { newConfig.setPhysicalDataset(new PhysicalDataset()); } } }
private String createPhysicalDSInHome(NamespaceService ns, String path, DatasetType datasetType) throws NamespaceException{ DatasetConfig datasetConfig = new DatasetConfig(); FilePath filePath = new FilePath(path); datasetConfig.setType(datasetType); datasetConfig.setFullPathList(filePath.toPathList()); datasetConfig.setName(filePath.getFileName().toString()); datasetConfig.setCreatedAt(System.currentTimeMillis()); datasetConfig.setTag(null); datasetConfig.setOwner("test_user"); datasetConfig.setPhysicalDataset(new PhysicalDataset()); ns.addOrUpdateDataset(filePath.toNamespaceKey(), datasetConfig); return datasetConfig.getTag(); }
private String createPhysicalDS(NamespaceService ns, String path, DatasetType datasetType) throws NamespaceException{ DatasetConfig datasetConfig = new DatasetConfig(); PhysicalDatasetPath physicalDatasetPath = new PhysicalDatasetPath(path); datasetConfig.setType(datasetType); datasetConfig.setFullPathList(physicalDatasetPath.toPathList()); datasetConfig.setName(physicalDatasetPath.getLeaf().getName()); datasetConfig.setCreatedAt(System.currentTimeMillis()); datasetConfig.setTag(null); datasetConfig.setOwner("test_user"); datasetConfig.setPhysicalDataset(new PhysicalDataset()); ns.addOrUpdateDataset(physicalDatasetPath.toNamespaceKey(), datasetConfig); return datasetConfig.getTag(); }
public static void addPhysicalDS(NamespaceService ns, String filePath, DatasetType type, byte[] datasetSchema) throws Exception { NamespaceKey datasetPath = new NamespaceKey(PathUtils.parseFullPath(filePath)); final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setName(datasetPath.getName()); datasetConfig.setType(type); final PhysicalDataset physicalDataset = new PhysicalDataset(); if (datasetSchema != null) { datasetConfig.setRecordSchema(io.protostuff.ByteString.copyFrom(datasetSchema)); } datasetConfig.setSchemaVersion(DatasetHelper.CURRENT_VERSION); datasetConfig.setPhysicalDataset(physicalDataset); ns.tryCreatePhysicalDataset(datasetPath, datasetConfig); }
protected static DatasetConfig addJson(DatasetPath path) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return nsService.getDataset(path.toNamespaceKey()); }
public void addJson(DatasetPath path, DatasetPath vdsPath) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); createDatasetFromParentAndSave(vdsPath, path.toPathString()); }
protected DatasetConfig addDataSet(DatasetPath path) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FOLDER) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.PARQUET).setCtime(1L).setOwner (DEFAULT_USERNAME)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return nsService.getDataset(path.toNamespaceKey()); } }