private boolean storeSchema(NamespaceKey key, DatasetConfig config) throws NamespaceException { try { namespace.addOrUpdateDataset(key, config); return true; } catch (final ConcurrentModificationException ex) { return false; } } }
DatasetConfig completeSave(DatasetConfig newConfig, List<DatasetSplit> splits, NamespaceAttribute... attributes) throws NamespaceException{ NamespaceKey key = new NamespaceKey(newConfig.getFullPathList()); systemUserNamespaceService.addOrUpdateDataset(key, newConfig, splits, attributes); updateListener.metadataUpdated(key); return newConfig; } }
void shallowSave(SourceTableDefinition accessor) throws NamespaceException{ NamespaceKey key = accessor.getName(); DatasetConfig shallow = new DatasetConfig(); shallow.setId(new EntityId().setId(UUID.randomUUID().toString())); shallow.setCreatedAt(System.currentTimeMillis()); shallow.setName(key.getName()); shallow.setFullPathList(key.getPathComponents()); shallow.setType(accessor.getType()); shallow.setSchemaVersion(DatasetHelper.CURRENT_VERSION); systemUserNamespaceService.addOrUpdateDataset(key, shallow); }
@Override public void upgrade(UpgradeContext context) throws Exception { final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider()); try { for (SourceConfig source : namespaceService.getSources()) { if (!"HIVE".equalsIgnoreCase(ConnectionReader.toType(source))) { continue; } System.out.printf(" Handling Hive source %s%n", source.getName()); for (NamespaceKey datasetPath : namespaceService.getAllDatasets(new NamespaceKey(source.getName()))) { final DatasetConfig datasetConfig = namespaceService.getDataset(datasetPath); if (datasetConfig.getReadDefinition() == null || datasetConfig.getReadDefinition().getExtendedProperty() == null) { continue; } System.out.printf(" Clearing read definition of table %s%n", datasetPath.getSchemaPath()); datasetConfig.setReadDefinition(null); namespaceService.addOrUpdateDataset(datasetPath, datasetConfig); } } } catch (NamespaceException e) { throw new RuntimeException("Hive121BasedInputSplits failed", e); } }
private void loadIfNecessary() { if(datasetPointer != null){ return; } SplitsPointer splitsPointer; if(datasetConfig.getReadDefinition() != null) { splitsPointer = DatasetSplitsPointer.of(ns, datasetConfig); } else { try{ final DatasetConfig newDatasetConfig = datasetAccessor.getDataset(); newDatasetConfig.setId(datasetConfig.getId()); newDatasetConfig.setTag(datasetConfig.getTag()); List<DatasetSplit> splits = datasetAccessor.getSplits(); ns.addOrUpdateDataset(getName(), newDatasetConfig, splits); datasetConfig = newDatasetConfig; splitsPointer = MaterializedSplitsPointer.of(splits, splits.size()); } catch (Exception e) { throw new RuntimeException(e); } } datasetPointer = new TableMetadataImpl(pluginId, datasetConfig, user, splitsPointer); }
private void saveInHomeSpace(NamespaceService namespaceService, SourceTableDefinition accessor, DatasetConfig nsConfig) { Preconditions.checkNotNull(nsConfig); final NamespaceKey key = new NamespaceKey(nsConfig.getFullPathList()); try{ // use key from namespace config DatasetConfig srcConfig = accessor.getDataset(); if (nsConfig.getId() == null) { nsConfig.setId(srcConfig.getId()); } // Merge namespace config with config obtained from underlying filesystem used to store user uploaded files. // Set schema, read definition and state from source accessor nsConfig.setRecordSchema(srcConfig.getRecordSchema()); nsConfig.setSchemaVersion(srcConfig.getSchemaVersion()); nsConfig.setReadDefinition(srcConfig.getReadDefinition()); // get splits from source List<DatasetSplit> splits = accessor.getSplits(); namespaceService.addOrUpdateDataset(key, nsConfig, splits); }catch(Exception ex){ logger.warn("Failure while retrieving and saving dataset {}.", key, ex); } }
public static void addFile(NamespaceService ns, List<String> path) throws Exception { NamespaceKey filePath = new NamespaceKey(path); final boolean isHome = path.get(0).startsWith("@"); final DatasetConfig ds = new DatasetConfig() .setType(isHome ? DatasetType.PHYSICAL_DATASET_HOME_FILE : DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig())); ns.addOrUpdateDataset(filePath, ds); }
namespace.addOrUpdateDataset(datasetKey, datasetConfig);
System.out.printf("compressed %d -> %d (%d%%)%n", original.length, compressed.length, ratio); namespaceService.addOrUpdateDataset(datasetPath, datasetConfig);
@Override public void updateView(List<String> path, String sql, List<String> sqlContext, NamespaceAttribute... attributes) { SecurityContext securityContext = getSecurityContext(); QueryExecutor executor = new QueryExecutor(jobsService, null, securityContext); DatasetVersion version = DatasetVersion.newVersion(); NamespaceKey namespaceKey = new NamespaceKey(path); try { DatasetConfig dataset = namespaceService.getDataset(namespaceKey); DatasetPath datasetPath = new DatasetPath(path); final VirtualDatasetUI virtualDataset = datasetService.get(datasetPath); Transformer transformer = new Transformer(contextService.get().get(), namespaceService, datasetService, executor, securityContext); TransformUpdateSQL transformUpdateSQL = new TransformUpdateSQL(); transformUpdateSQL.setSql(sql); transformUpdateSQL.setSqlContextList(sqlContext); VirtualDatasetUI virtualDatasetUI = transformer.transformWithExtract(version, datasetPath, virtualDataset, transformUpdateSQL); // copy over VirtualDatasetUI values to the pre-existing DatasetConfig dataset.setVirtualDataset(DatasetsUtil.toVirtualDataset(virtualDatasetUI)); dataset.setRecordSchema(virtualDatasetUI.getRecordSchema()); namespaceService.addOrUpdateDataset(namespaceKey, dataset, attributes); } catch (Exception e) { throw Throwables.propagate(e); } }
public void put(VirtualDatasetUI ds, NamespaceAttribute... attributes) throws DatasetNotFoundException, NamespaceException { DatasetPath path = new DatasetPath(ds.getFullPathList()); validatePath(path); validate(path, ds); DatasetConfig datasetConfig = toVirtualDatasetVersion(ds).getDataset(); namespaceService.addOrUpdateDataset(path.toNamespaceKey(), datasetConfig, attributes); ds.setId(datasetConfig.getId().getId()); ds.setSavedTag(datasetConfig.getTag()); // Update this version of dataset with new occ version of dataset config from namespace. putVersion(ds); }
DatasetConfig config = table.getDatasetConfig(); config.getPhysicalDataset().setAllowApproxStats(sqlSetApprox.isEnable()); namespaceService.addOrUpdateDataset(path, config); return singletonList(successful(String.format("Successfully updated table '%s' from namespace.", table.getPath()))); } catch (NamespaceNotFoundException ex) {
private static String addJsonTable(String tableName, String... jsonData) throws Exception { final File file = temp.newFile(tableName); final String dataFile = file.getAbsolutePath(); //TODO write each record in a separate file, so we can cause a union type for example try (PrintWriter writer = new PrintWriter(file)) { for (String record : jsonData) { writer.println(record); } } final DatasetPath path = new DatasetPath(ImmutableList.of("dfs", dataFile)); final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return dataFile; } }
private String createPhysicalDSInHome(NamespaceService ns, String path, DatasetType datasetType) throws NamespaceException{ DatasetConfig datasetConfig = new DatasetConfig(); FilePath filePath = new FilePath(path); datasetConfig.setType(datasetType); datasetConfig.setFullPathList(filePath.toPathList()); datasetConfig.setName(filePath.getFileName().toString()); datasetConfig.setCreatedAt(System.currentTimeMillis()); datasetConfig.setTag(null); datasetConfig.setOwner("test_user"); datasetConfig.setPhysicalDataset(new PhysicalDataset()); ns.addOrUpdateDataset(filePath.toNamespaceKey(), datasetConfig); return datasetConfig.getTag(); }
private String createPhysicalDS(NamespaceService ns, String path, DatasetType datasetType) throws NamespaceException{ DatasetConfig datasetConfig = new DatasetConfig(); PhysicalDatasetPath physicalDatasetPath = new PhysicalDatasetPath(path); datasetConfig.setType(datasetType); datasetConfig.setFullPathList(physicalDatasetPath.toPathList()); datasetConfig.setName(physicalDatasetPath.getLeaf().getName()); datasetConfig.setCreatedAt(System.currentTimeMillis()); datasetConfig.setTag(null); datasetConfig.setOwner("test_user"); datasetConfig.setPhysicalDataset(new PhysicalDataset()); ns.addOrUpdateDataset(physicalDatasetPath.toNamespaceKey(), datasetConfig); return datasetConfig.getTag(); }
public static void addDS(NamespaceService ns, String name) throws Exception { final NamespaceKey dsPath = new NamespaceKey(PathUtils.parseFullPath(name)); final DatasetConfig ds = new DatasetConfig(); final VirtualDataset vds = new VirtualDataset(); vds.setVersion(DatasetVersion.newVersion()); ds.setType(DatasetType.VIRTUAL_DATASET); ds.setVirtualDataset(vds); ds.setFullPathList(dsPath.getPathComponents()); ds.setName(dsPath.getName()); ns.addOrUpdateDataset(dsPath, ds); }
context.getNamespaceService().addOrUpdateDataset(key, updatedDatasetConfig); } catch (NamespaceException e) {
protected static DatasetConfig addJson(DatasetPath path) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return nsService.getDataset(path.toNamespaceKey()); }
public void addJson(DatasetPath path, DatasetPath vdsPath) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FILE) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setTag(null) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.JSON)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); createDatasetFromParentAndSave(vdsPath, path.toPathString()); }
protected DatasetConfig addDataSet(DatasetPath path) throws Exception { final DatasetConfig dataset = new DatasetConfig() .setType(DatasetType.PHYSICAL_DATASET_SOURCE_FOLDER) .setFullPathList(path.toPathList()) .setName(path.getLeaf().getName()) .setCreatedAt(System.currentTimeMillis()) .setOwner(DEFAULT_USERNAME) .setPhysicalDataset(new PhysicalDataset() .setFormatSettings(new FileConfig().setType(FileType.PARQUET).setCtime(1L).setOwner (DEFAULT_USERNAME)) ); final NamespaceService nsService = getNamespaceService(); nsService.addOrUpdateDataset(path.toNamespaceKey(), dataset); return nsService.getDataset(path.toNamespaceKey()); } }