/** * returns a Qlik load script for the dataset * @return * @throws DatasetNotFoundException * @throws NamespaceException */ @GET @Produces(TEXT_PLAIN_QLIK_APP) public DatasetConfig get() throws DatasetNotFoundException, NamespaceException { return namespaceService.getDataset(datasetPath.toNamespaceKey()); } }
@VisibleForTesting public Iterable<ReflectionGoal> getReflectionGoals(final NamespaceKey path, final String reflectionName) { try { DatasetConfig config = namespaceService.get().getDataset(path); return FluentIterable.from(getReflectionsByDatasetId(config.getId().getId())).filter(new Predicate<ReflectionGoal>() { @Override public boolean apply(ReflectionGoal input) { return reflectionName.equals(input.getName()); }}); }catch(NamespaceException ex) { throw Throwables.propagate(ex); } }
@Override public Iterable<ReflectionGoal> getReflectionsByDatasetPath(NamespaceKey path) { try { DatasetConfig config = namespaceService.get().getDataset(path); return getReflectionsByDatasetId(config.getId().getId()); }catch(NamespaceException ex) { throw Throwables.propagate(ex); } }
@GET @Path("file_format/{path: .*}") @Produces(MediaType.APPLICATION_JSON) public FileFormatUI getFormatSettings(@PathParam("path") String path) throws FileNotFoundException, HomeNotFoundException, NamespaceException { FilePath filePath = FilePath.fromURLPath(homeName, path); final FileConfig fileConfig = toFileConfig(namespaceService.getDataset(filePath.toNamespaceKey())); return new FileFormatUI(FileFormat.getForFile(fileConfig), filePath); }
@DELETE @Path("file/{path: .*}") @Produces(MediaType.APPLICATION_JSON) public void deleteFile(@PathParam("path") String path, @QueryParam("version") String version) throws NamespaceException, DACException { FilePath filePath = FilePath.fromURLPath(homeName, path); if (version == null) { throw new ClientErrorException("missing version parameter"); } try { catalogServiceHelper.deleteHomeDataset(namespaceService.getDataset(filePath.toNamespaceKey()), version); } catch (IOException ioe) { throw new DACException("Error deleting to file at " + filePath, ioe); } }
@Override public void observedSchemaChange(List<String> key, BatchSchema expectedSchema, BatchSchema newlyObservedSchema, SchemaChangeMutator schemaChangeMutator) { NamespaceKey namespaceKey = new NamespaceKey(key); boolean success; try { do { DatasetConfig oldConfig = namespace.getDataset(namespaceKey); DatasetConfig newConfig = schemaChangeMutator.updateForSchemaChange(oldConfig, expectedSchema, newlyObservedSchema); success = storeSchema(namespaceKey, newConfig); } while (!success); } catch (NamespaceException e) { throw new RuntimeException(e); } }
private void verifyDSNotInNamespace(NamespaceService ns, NamespaceKey nsKey) throws NamespaceException { try { ns.getDataset(nsKey); fail("getDataset didn't throw exception"); } catch (NamespaceNotFoundException nfe) { } }
public PhysicalDatasetConfig getFilesystemPhysicalDataset(NamespacePath path, DatasetType type) throws NamespaceException { try { return toPhysicalDatasetConfig(namespaceService.getDataset(path.toNamespaceKey())); } catch (NamespaceNotFoundException nse) { throw new PhysicalDatasetNotFoundException(path, type, nse); } }
public static VirtualDatasetUI getHeadVersion(DatasetPath datasetPath, NamespaceService namespaceService, DatasetVersionMutator datasetService) throws NamespaceException, DatasetVersionNotFoundException { DatasetConfig dsConfig = namespaceService.getDataset(datasetPath.toNamespaceKey()); return datasetService.getVersion(datasetPath, dsConfig.getVirtualDataset().getVersion()); }
private void assertDatasetSchemasNotDefined(List<NamespaceKey> datasetKeys) throws Exception { for (NamespaceKey datasetKey : datasetKeys) { DatasetConfig dsConfig = namespaceService.getDataset(datasetKey); assertEquals(dsConfig.getRecordSchema(), null); } }
public boolean isPhysicalDataset(SourceName sourceName, SourceFolderPath folderPath) { try { DatasetConfig ds = namespaceService.getDataset(new PhysicalDatasetPath(folderPath).toNamespaceKey()); return DatasetHelper.isPhysicalDataset(ds.getType()); } catch (NamespaceException nse) { logger.debug("Error while checking physical dataset in source {} for folder {}, error {}", sourceName.getName(), folderPath.toPathString(), nse.toString()); return false; } }
@Override public Iterable<ExternalReflection> getExternalReflectionByDatasetPath(List<String> datasetPath) { DatasetConfig datasetConfig; try { datasetConfig = namespaceService.get().getDataset(new NamespaceKey(datasetPath)); } catch (NamespaceException e) { throw UserException.validationError(e).build(logger); } if (datasetConfig == null) { throw UserException.validationError().message(String.format("Dataset %s not found", quotedCompound(datasetPath))).build(logger); } return externalReflectionStore.findByDatasetId(datasetConfig.getId().getId()); }
private void deleteDataset(List<String> path) throws NamespaceException { final NamespaceKey key = new NamespaceKey(path); try { DatasetConfig oldDataset = namespaceService.getDataset(key); namespaceService.deleteDataset(key, oldDataset.getTag()); } catch (NamespaceNotFoundException e) { // Ignore } } }
private boolean schemaChanged(MaterializationDescriptor old, Materialization materialization) { if (namespaceService == null) { return false; } try { //TODO is this enough ? shouldn't we use the dataset hash instead ?? final NamespaceKey matKey = new NamespaceKey(ReflectionUtils.getMaterializationPath(materialization)); ByteString schemaString = namespaceService.getDataset(matKey).getRecordSchema(); BatchSchema newSchema = BatchSchema.deserialize(schemaString); BatchSchema oldSchema = ((CachedMaterializationDescriptor) old).getMaterialization().getSchema(); return !oldSchema.equals(newSchema); } catch (NamespaceException e) { return true; } }
@Test public void testReadSignatures() throws Exception { getSabotContext().getCatalogService().refreshSource(new NamespaceKey("hive"), CatalogService.REFRESH_EVERYTHING_NOW, UpdateType.FULL); NamespaceService ns = getSabotContext().getNamespaceService(SystemUser.SYSTEM_USERNAME); assertEquals(2, getCachedEntities(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.db1.kv_db1")))).size()); assertEquals(2, getCachedEntities(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.db1.avro")))).size()); assertEquals(2, getCachedEntities(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.\"default\".dummy")))).size()); assertEquals(2, getCachedEntities(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.skipper.kv_parquet_large")))).size()); assertEquals(3, getCachedEntities(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.\"default\".readtest")))).size()); assertEquals(3, getCachedEntities(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.\"default\".readtest_parquet")))).size()); assertEquals(10, getCachedEntities(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.\"default\".kv_parquet")))).size()); assertEquals(54, getCachedEntities(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.\"default\".partition_with_few_schemas")))).size()); assertEquals(56, getCachedEntities(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.\"default\".partition_pruning_test")))).size()); }
public void renameDataset(DatasetPath ds, DatasetPath dsNew) { NamespaceKey key = ds.toNamespaceKey(); DatasetConfig config; try { config = getNamespaceService().getDataset(key); if (config != null) { getNamespaceService().renameDataset(key, dsNew.toNamespaceKey()); } }catch (NamespaceException e) { } }
public void deleteDataset(DatasetPath ds) { NamespaceKey key = ds.toNamespaceKey(); DatasetConfig config; try { config = getNamespaceService().getDataset(key); if (config != null) { getNamespaceService().deleteDataset(key, config.getTag()); } }catch (NamespaceException e) { } }
protected static void requestRefresh(NamespaceKey datasetKey) throws NamespaceException { DatasetConfig dataset = getNamespaceService().getDataset(datasetKey); getReflectionService().requestRefresh(dataset.getId().getId()); }
private void assertDatasetSchemasDefined(List<NamespaceKey> datasetKeys) throws Exception { for (NamespaceKey datasetKey : datasetKeys) { DatasetConfig dsConfig = namespaceService.getDataset(datasetKey); BatchSchema schema = BatchSchema.deserialize(dsConfig.getRecordSchema()); assertEquals(schema.getFieldCount(), 1); assertEquals(schema.getColumn(0).getName(), "string"); assertEquals(schema.getColumn(0).getType(), ArrowType.Utf8.INSTANCE); } }
@Test public void testCreateDatasets() throws Exception { expectSuccess(getBuilder(getAPIv2().path("space/spaceCreateDataset")).buildPut(Entity.json(new Space(null, "spaceCreateDataset", null, null, null, 0, null))), Space.class); DatasetPath datasetPath = new DatasetPath("spaceCreateDataset.ds1"); DatasetUI ds1 = createDatasetFromSQLAndSave(datasetPath, "select s.s_name from cp.\"tpch/supplier.parquet\" s", asList("cp")); DatasetConfig dataset = l(NamespaceService.class).getDataset(datasetPath.toNamespaceKey()); assertEquals(ds1.getVersion(), dataset.getTag()); getDataset(getDatasetPath(ds1)); DatasetUI ds2 = createDatasetFromParentAndSave(new DatasetPath("spaceCreateDataset.ds3"), getDatasetPath(ds1).toPathString()); getDataset(getDatasetPath(ds2)); }