public NamespaceNotFoundException(NamespaceKey key, String message) { super(message + ": " + key.toString()); this.key = key; }
public String getDefaultSchemaName() { return defaultSchemaPath == null ? "" : defaultSchemaPath.toString(); }
@Override public byte[] convert(NamespaceKey v) { return StringSerializer.INSTANCE.convert(v.toString()); }
@Override public String toJson(NamespaceKey v) throws IOException { return StringSerializer.INSTANCE.toJson(v.toString()); }
@Override public String toString(){ return new NamespaceKey(Iterables.getOnlyElement(getReferencedTables())).toString(); }
@Override public String toString() { return format("%s : %s", type.toString(), new NamespaceKey(fullPath).toString()); }
private SearchQuery getFilter(final NamespaceKey datasetPath, final DatasetVersion datasetVersion, final String user) { Preconditions.checkNotNull(datasetPath); ImmutableList.Builder<SearchQuery> builder = ImmutableList.<SearchQuery> builder() .add(SearchQueryUtils.newTermQuery(ALL_DATASETS, datasetPath.toString())) .add(JobIndexKeys.UI_EXTERNAL_JOBS_FILTER); if (datasetVersion != null) { builder.add(SearchQueryUtils.newTermQuery(DATASET_VERSION, datasetVersion.getVersion())); } if (user != null) { builder.add((SearchQueryUtils.newTermQuery(USER, user))); } return SearchQueryUtils.and(builder.build()); }
/** * Get count of datasets depending on given dataset * @param path path of saved dataset * @return count of all descendants. * @throws NamespaceException */ public int getDescendantsCount(NamespaceKey path) { try (TimedBlock b = Timer.time("getDescendantCounts")) { return namespaceService.getCounts(SearchQueryUtils.newTermQuery(DATASET_ALLPARENTS, path.toString())).get(0); } catch(NamespaceException e) { logger.error("Failed to get descendant counts for path " + path); return 0; } }
private String getJobsUrl() { final NamespaceKey datasetPath = new NamespaceKey(fullPath); final JobFilters jobFilters = new JobFilters() .addFilter(JobIndexKeys.ALL_DATASETS, datasetPath.toString()) .addFilter(JobIndexKeys.QUERY_TYPE, JobIndexKeys.UI, JobIndexKeys.EXTERNAL); return jobFilters.toUrl(); }
.message("Access denied reading dataset %s.", namespaceKey.toString()) .build(logger);
protected CatalogEntity createDataset(Dataset dataset, NamespaceAttribute... attributes) throws NamespaceException { validateDataset(dataset); // only handle VDS Preconditions.checkArgument(dataset.getType() != Dataset.DatasetType.PHYSICAL_DATASET, "Physical Datasets can only be created by promoting other entities."); Preconditions.checkArgument(dataset.getId() == null, "Dataset id is immutable."); // verify we can save NamespaceKey topLevelKey = new NamespaceKey(dataset.getPath().get(0)); NamespaceKey parentNamespaceKey = new NamespaceKey(dataset.getPath().subList(0, dataset.getPath().size() - 1)); NamespaceKey namespaceKey = new NamespaceKey(dataset.getPath()); Preconditions.checkArgument(namespaceService.exists(parentNamespaceKey), String.format("Dataset parent path [%s] doesn't exist.", parentNamespaceKey.toString())); // can only create VDs in a space or home NameSpaceContainer entity = getNamespaceEntity(topLevelKey); List<NameSpaceContainer.Type> types = Arrays.asList(NameSpaceContainer.Type.SPACE, NameSpaceContainer.Type.HOME); Preconditions.checkArgument(types.contains(entity.getType()), "Virtual datasets can only be saved into spaces or home space."); sabotContext.getViewCreator(context.getUserPrincipal().getName()).createView(dataset.getPath(), dataset.getSql(), dataset.getSqlContext(), attributes); DatasetConfig created = namespaceService.getDataset(namespaceKey); return getDatasetFromConfig(created, null); }
/** * Get list of dataset paths depending on given dataset * @param path path of saved dataset * @return dataset paths of descendants. * @throws NamespaceException */ public Iterable<DatasetPath> getDescendants(DatasetPath path) throws NamespaceException { FindByCondition condition = new FindByCondition() .setCondition(SearchQueryUtils.newTermQuery(DATASET_ALLPARENTS, path.toNamespaceKey().toString())) .setLimit(1000); return Iterables.transform(namespaceService.find(condition), new Function<Entry<NamespaceKey, NameSpaceContainer>, DatasetPath>() { @Override public DatasetPath apply(Entry<NamespaceKey, NameSpaceContainer> input) { return new DatasetPath(input.getKey().getPathComponents()); } }); }
.addContainsFilter(folderPath.toNamespaceKey().toString()) .addFilter(JobIndexKeys.QUERY_TYPE, "UI", "EXTERNAL"); links.put("jobs", jobFilters.toUrl());
@Override public TranslatableTable apply(final List<Object> arguments) { try { SourceTableDefinition definition = plugin.getDatasetWithOptions(new NamespaceKey (tableSchemaPath), new TableInstance(sig, arguments), schemaConfig.getIgnoreAuthErrors(), schemaConfig.getUserName()); if(definition == null){ throw UserException.validationError().message("Unable to read table %s using provided options.", new NamespaceKey(tableSchemaPath).toString()).build(logger); } return new MaterializedDatasetTable(plugin, schemaConfig.getUserName(), new QuietAccessor(definition)); } catch (Exception e) { throw new RuntimeException(e); } }
final SchemaMerger merger = new SchemaMerger(new NamespaceKey(dataset.getFullPathList()).toString()); final BatchSchema schema; if(oldConfig == null || DatasetHelper.getSchemaBytes(oldConfig) == null){
@Override protected SourceTableDefinition getDatasetWithFormat(NamespaceKey datasetPath, DatasetConfig oldConfig, FormatPluginConfig formatPluginConfig, DatasetRetrievalOptions retrievalOptions, String user) throws Exception { try{ final FileSystemWrapper fs = getFs(); final DatasetConfig datasetConfig = getContext().getNamespaceService(SystemUser.SYSTEM_USERNAME).getDataset(datasetPath); if (!(datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FILE || datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FOLDER)) { throw new IllegalArgumentException(format("Table %s does not belong to home space", datasetPath.toString())); } final FormatPlugin formatPlugin = formatCreator.newFormatPlugin(formatPluginConfig); return getDataset(datasetPath, oldConfig, formatPlugin, fs, datasetConfig.getPhysicalDataset().getFormatSettings()); } catch (NamespaceNotFoundException nfe){ if(formatPluginConfig == null) { // a home file can only be read from the namespace or using a format options. Without either, it is invalid, return nothing. return null; } return super.getDatasetWithFormat(new NamespaceKey(relativePath(datasetPath.getPathComponents(), getConfig().getPath())), oldConfig, formatPluginConfig, retrievalOptions, null); } }
@Override public SourceTableDefinition getDataset(NamespaceKey datasetPath, DatasetConfig oldConfig, DatasetRetrievalOptions retrievalOptions) throws Exception { if (datasetPath.size() <= 1) { return null; } PhysicalDataset physicalDataset = oldConfig == null ? null : oldConfig.getPhysicalDataset(); FormatPluginConfig pluginConfig = null; try { final FileSystemWrapper fs = getFs(); final FileConfig fileConfig; if(physicalDataset != null && physicalDataset.getFormatSettings() != null){ fileConfig = physicalDataset.getFormatSettings(); }else{ final DatasetConfig datasetConfig =getContext().getNamespaceService(SystemUser.SYSTEM_USERNAME).getDataset(datasetPath); if (!(datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FILE || datasetConfig.getType() == DatasetType.PHYSICAL_DATASET_HOME_FOLDER)) { throw new IllegalArgumentException(format("Table %s does not belong to home space", datasetPath.toString())); } fileConfig = datasetConfig.getPhysicalDataset().getFormatSettings(); } pluginConfig = PhysicalDatasetUtils.toFormatPlugin(fileConfig, Collections.<String>emptyList()); final FormatPlugin formatPlugin = formatCreator.newFormatPlugin(pluginConfig); return getDataset(datasetPath, oldConfig, formatPlugin, fs, fileConfig); } catch (NamespaceNotFoundException nfe){ FormatPluginConfig formatPluginConfig = null; if(physicalDataset != null && physicalDataset.getFormatSettings() != null){ formatPluginConfig = PhysicalDatasetUtils.toFormatPlugin(physicalDataset.getFormatSettings(), Collections.<String>emptyList()); } return getDatasetWithFormat(datasetPath, oldConfig, formatPluginConfig, retrievalOptions, null); } }
SchemaMerger merger = new SchemaMerger(new NamespaceKey(oldConfig.getFullPathList()).toString());
@Test public void testReapplyDataset() { expectSuccess(getBuilder(getAPIv2().path("space/space1")).buildPut(Entity.json(new Space(null, "space1", null, null, null, 0, null))), Space.class); expectSuccess(getBuilder(getAPIv2().path("space/space2")).buildPut(Entity.json(new Space(null, "space2", null, null, null, 0, null))), Space.class); DatasetPath d1Path = new DatasetPath("space1.ds1"); DatasetPath d2Path = new DatasetPath("space1.ds2"); DatasetUI d1 = createDatasetFromSQLAndSave(d1Path, "select s_name, s_phone from cp.\"tpch/supplier.parquet\"", asList("cp")); DatasetUI d2 = createDatasetFromSQLAndSave(d2Path, "select s_name, s_phone from cp.\"tpch/supplier.parquet\"", asList("cp")); doc("creating untitled from existing dataset"); final DatasetUI d3 = createDatasetFromParent(d1Path.toNamespaceKey().toString()).getDataset(); doc("verifying untitled ds"); final DatasetUI d4 = getVersionedDataset(getDatasetVersionPath(d3)); doc("applying transform to untitled ds"); final DatasetUI d5 = transform(d4, new TransformField("s_phone", "s_phone_json", false, new FieldConvertToJSON().wrap())).getDataset(); assertContains("cast(convert_from(convert_to(s_phone, 'json'), 'utf8') as varchar) as s_phone_json", d5.getSql().toLowerCase()); doc("verifying transformed ds"); final DatasetUI d6 = getVersionedDataset(getDatasetVersionPath(d5)); assertEquals(d6.getFullPath().get(0), "tmp"); doc("reapplying transform from untitled to ds1"); InitialPreviewResponse reapplyResult = reapply(getDatasetVersionPath(d6)); final DatasetUI d7 = reapplyResult.getDataset(); assertEquals(Arrays.asList("space1", "ds1"), d7.getFullPath()); assertEquals(reapplyResult.getHistory().getItems().size(), 1); // should be only data set creation. Transformation should be ignored HistoryItem firstHistoryItem = reapplyResult.getHistory().getItems().get(0); assertEquals(firstHistoryItem.getDataset(), d1Path); assertEquals(firstHistoryItem.getDatasetVersion(), d1.getDatasetVersion()); doc("should fail to edit original sql for dataset that is created from sql"); expectError(CLIENT_ERROR, reapplyInvocation(getDatasetVersionPath(d2)), UserExceptionMapper.ErrorMessageWithContext.class); }