public NamespacePath(String path) { this(PathUtils.parseFullPath(path)); }
/** * Convert fully or partially dotted path to file system path. * a.b.c -> /a/b/c * a.`b/c` -> /a/b/c * @param path a fully or partially dotted path * @return {@code fs Path} */ public static Path toFSPath(String path) { final List<String> pathComponents = Lists.newArrayList(); for (String component: parseFullPath(path)) { if (component.contains(SLASH)) { pathComponents.addAll(toPathComponents(new Path(component))); } else { pathComponents.add(component); } } return toFSPath(pathComponents); }
public static NamespaceInternalKey parseKey(byte[] keyBytes) { String path = extractKey(keyBytes, false); return new NamespaceInternalKey(new NamespaceKey(PathUtils.parseFullPath(path))); }
final List<String> spacePathComponents = PathUtils.parseFullPath(path.getRoot()); cleanedPathComponents.addAll(spacePathComponents); List<String> pathComponents = path.getPathComponents();
public static Map<String, NameSpaceContainer> listFolder(NamespaceService ns, String parent) throws Exception { Map<String, NameSpaceContainer> children = new HashMap<>(); for (NameSpaceContainer container : ns.list(new NamespaceKey(PathUtils.parseFullPath(parent)))) { children.put(PathUtils.constructFullPath(container.getFullPathList()), container); } return children; }
private void setScanStats(CoreOperatorType operatorType, OperatorProfile operatorProfile, MajorFragmentProfile majorFragment) { final String operatorId = createOperatorKey(majorFragment.getMajorFragmentId(), operatorProfile.getOperatorId()); long inputBytes = 0L; long inputRecords = 0L; if (operatorProfile.getInputProfileList() != null) { for (StreamProfile stream : operatorProfile.getInputProfileList()) { inputRecords += stream.getRecords(); inputBytes += stream.getSize(); } addInputBytesAndRecords(inputBytes, inputRecords); } if (operatorToTable.containsKey(operatorId)) { // TODO check if its fs based or table based using operatorType final String tableName = operatorToTable.get(operatorId); TableDatasetProfile tableDatasetProfile = tableDatasetProfileMap.get(tableName); if (tableDatasetProfile == null) { tableDatasetProfile = new TableDatasetProfile(); tableDatasetProfile.setDatasetProfile(new CommonDatasetProfile().setDatasetPathsList(Lists.<DatasetPathUI>newArrayList())); tableDatasetProfileMap.put(tableName, tableDatasetProfile); } setCommonDatasetProfile(tableDatasetProfile.getDatasetProfile(), operatorProfile, majorFragment, inputBytes, inputRecords, PathUtils.parseFullPath(tableName)); tableDatasetProfile.setPushdownQuery(null); } }
private static NamespaceInternalKey newKey(String path) { return new NamespaceInternalKey(new NamespaceKey(PathUtils.parseFullPath(path))); } }
@Test public void testReadSignatures() throws Exception { getSabotContext().getCatalogService().refreshSource(new NamespaceKey("hive"), CatalogService.REFRESH_EVERYTHING_NOW, UpdateType.FULL); NamespaceService ns = getSabotContext().getNamespaceService(SystemUser.SYSTEM_USERNAME); assertEquals(2, getCachedEntities(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.db1.kv_db1")))).size()); assertEquals(2, getCachedEntities(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.db1.avro")))).size()); assertEquals(2, getCachedEntities(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.\"default\".dummy")))).size()); assertEquals(2, getCachedEntities(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.skipper.kv_parquet_large")))).size()); assertEquals(3, getCachedEntities(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.\"default\".readtest")))).size()); assertEquals(3, getCachedEntities(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.\"default\".readtest_parquet")))).size()); assertEquals(10, getCachedEntities(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.\"default\".kv_parquet")))).size()); assertEquals(54, getCachedEntities(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.\"default\".partition_with_few_schemas")))).size()); assertEquals(56, getCachedEntities(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.\"default\".partition_pruning_test")))).size()); }
} else { final List<String> tableNamePathComponents = PathUtils.parseFullPath(tableName); tableName = tableNamePathComponents.remove(tableNamePathComponents.size() - 1);
public static void addFolder(NamespaceService ns, String name) throws Exception { final FolderConfig folder = new FolderConfig(); final NamespaceKey folderPath = new NamespaceKey(PathUtils.parseFullPath(name)); folder.setName(folderPath.getName()); folder.setFullPathList(folderPath.getPathComponents()); ns.addOrUpdateFolder(folderPath, folder); }
ns.renameDataset(new NamespaceKey(PathUtils.parseFullPath("s.b.ds2")), new NamespaceKey(PathUtils.parseFullPath("s.b.ds22"))); items = listFolder(ns, "s.b"); assertEquals(1, items.size()); assertTrue(items.containsKey("s.b.ds22")); ns.renameDataset(new NamespaceKey(PathUtils.parseFullPath("s.a.c.ds3")), new NamespaceKey(PathUtils.parseFullPath("s.a.c.ds33"))); items = listFolder(ns, "s.a.c"); assertEquals(1, items.size()); ns.renameDataset(new NamespaceKey(PathUtils.parseFullPath(("@blue.a.c.file1"))), new NamespaceKey(PathUtils.parseFullPath(("s.a.c.file1DOTjson")))); Assert.fail("renames on physical datasets should not be allowed"); } catch (final UserException ex) { ns.renameDataset(new NamespaceKey(PathUtils.parseFullPath("s.a.c.ds33")), new NamespaceKey(PathUtils.parseFullPath("L.F.ds33r"))); items = listFolder(ns, "@blue.a.c"); assertEquals(1, items.size());
@Test public void testCheckReadSignature() throws Exception { getSabotContext().getCatalogService().refreshSource(new NamespaceKey("hive"), CatalogService.REFRESH_EVERYTHING_NOW, UpdateType.FULL); NamespaceService ns = getSabotContext().getNamespaceService(SystemUser.SYSTEM_USERNAME); DatasetConfig datasetConfig = ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.db1.kv_db1"))); assertEquals(UpdateStatus.UNCHANGED, getSabotContext().getCatalogService().getSource("hive").checkReadSignature( datasetConfig.getReadDefinition().getReadSignature(), datasetConfig, DatasetRetrievalOptions.DEFAULT).getStatus()); datasetConfig = ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.\"default\".partition_with_few_schemas"))); assertEquals(UpdateStatus.UNCHANGED, getSabotContext().getCatalogService().getSource("hive").checkReadSignature( datasetConfig.getReadDefinition().getReadSignature(), datasetConfig, DatasetRetrievalOptions.DEFAULT).getStatus()); new File(hiveTest.getWhDir() + "/db1.db/kv_db1", "000000_0").setLastModified(System.currentTimeMillis()); File newFile = new File(hiveTest.getWhDir() + "/partition_with_few_schemas/c=1/d=1/e=1/", "empty_file"); try { newFile.createNewFile(); datasetConfig = ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.db1.kv_db1"))); assertEquals(UpdateStatus.CHANGED, getSabotContext().getCatalogService().getSource("hive").checkReadSignature( datasetConfig.getReadDefinition().getReadSignature(), datasetConfig, DatasetRetrievalOptions.DEFAULT).getStatus()); datasetConfig = ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("hive.\"default\".partition_with_few_schemas"))); assertEquals(UpdateStatus.CHANGED, getSabotContext().getCatalogService().getSource("hive").checkReadSignature( datasetConfig.getReadDefinition().getReadSignature(), datasetConfig, DatasetRetrievalOptions.DEFAULT).getStatus()); } finally { newFile.delete(); } }
@Test public void testCheckHasPermission() throws Exception { getSabotContext().getCatalogService().refreshSource(new NamespaceKey("hive"), CatalogService.REFRESH_EVERYTHING_NOW, UpdateType.FULL); NamespaceService ns = getSabotContext().getNamespaceService(SystemUser.SYSTEM_USERNAME); NamespaceKey dataset = new NamespaceKey(PathUtils.parseFullPath("hive.db1.kv_db1")); DatasetConfig datasetConfig = ns.getDataset(dataset); assertTrue(getSabotContext().getCatalogService().getSource("hive").hasAccessPermission(ImpersonationUtil.getProcessUserName(), dataset, datasetConfig)); final Path tableFile = new Path(hiveTest.getWhDir() + "/db1.db/kv_db1/000000_0"); final Path tableDir = new Path(hiveTest.getWhDir() + "/db1.db/kv_db1"); final FileSystem localFs = FileSystem.getLocal(new Configuration()); try { // no read on file localFs.setPermission(tableFile, new FsPermission(FsAction.WRITE_EXECUTE, FsAction.WRITE_EXECUTE, FsAction.WRITE_EXECUTE)); assertFalse(getSabotContext().getCatalogService().getSource("hive").hasAccessPermission(ImpersonationUtil.getProcessUserName(), dataset, datasetConfig)); } finally { localFs.setPermission(tableFile, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); } try { // no exec on dir localFs.setPermission(tableDir, new FsPermission(FsAction.READ_WRITE, FsAction.READ_WRITE, FsAction.READ_WRITE)); assertFalse(getSabotContext().getCatalogService().getSource("hive").hasAccessPermission(ImpersonationUtil.getProcessUserName(), dataset, datasetConfig)); } finally { localFs.setPermission(tableDir, new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL)); } }
public static void addPhysicalDS(NamespaceService ns, String filePath, DatasetType type, byte[] datasetSchema) throws Exception { NamespaceKey datasetPath = new NamespaceKey(PathUtils.parseFullPath(filePath)); final DatasetConfig datasetConfig = new DatasetConfig(); datasetConfig.setName(datasetPath.getName()); datasetConfig.setType(type); final PhysicalDataset physicalDataset = new PhysicalDataset(); if (datasetSchema != null) { datasetConfig.setRecordSchema(io.protostuff.ByteString.copyFrom(datasetSchema)); } datasetConfig.setSchemaVersion(DatasetHelper.CURRENT_VERSION); datasetConfig.setPhysicalDataset(physicalDataset); ns.tryCreatePhysicalDataset(datasetPath, datasetConfig); }
public static void addDS(NamespaceService ns, String name) throws Exception { final NamespaceKey dsPath = new NamespaceKey(PathUtils.parseFullPath(name)); final DatasetConfig ds = new DatasetConfig(); final VirtualDataset vds = new VirtualDataset(); vds.setVersion(DatasetVersion.newVersion()); ds.setType(DatasetType.VIRTUAL_DATASET); ds.setVirtualDataset(vds); ds.setFullPathList(dsPath.getPathComponents()); ds.setName(dsPath.getName()); ns.addOrUpdateDataset(dsPath, ds); }
@Test public void testDataSetSchema() throws Exception { try( final KVStoreProvider kvstore = new LocalKVStoreProvider(DremioTest.CLASSPATH_SCAN_RESULT, null, true, false); ) { kvstore.start(); final NamespaceService ns = new NamespaceServiceImpl(kvstore); Field field1 = new Field("a", true, new Int(32, true), null); Field child1 = new Field("c", true, Utf8.INSTANCE, null); Field field2 = new Field("b", true, Struct.INSTANCE, ImmutableList.of(child1)); Schema schema = new Schema(ImmutableList.of(field1, field2)); FlatBufferBuilder builder = new FlatBufferBuilder(); schema.getSchema(builder); builder.finish(schema.getSchema(builder)); addSource(ns, "s"); addPhysicalDS(ns, "s.foo", builder.sizedByteArray()); ByteBuffer bb = ByteBuffer.wrap(DatasetHelper.getSchemaBytes(ns.getDataset(new NamespaceKey(PathUtils.parseFullPath("s.foo")))).toByteArray()); Schema returnedSchema = Schema.convertSchema(org.apache.arrow.flatbuf.Schema.getRootAsSchema(bb)); assertEquals(schema, returnedSchema); } }
new NamespaceKey(PathUtils.parseFullPath("hive.\"default\".foo_bar")), Type.DATASET)); assertFalse(getSabotContext().getNamespaceService(SystemUser.SYSTEM_USERNAME).exists( new NamespaceKey(PathUtils.parseFullPath("hive.foo_bar")), Type.DATASET)); new NamespaceKey(PathUtils.parseFullPath("hive.foo_bar")), Type.DATASET)); new NamespaceKey(PathUtils.parseFullPath("hive.\"default\".foo_bar")), Type.DATASET)); assertFalse(getSabotContext().getNamespaceService(SystemUser.SYSTEM_USERNAME).exists( new NamespaceKey(PathUtils.parseFullPath("hive.foo_bar")), Type.DATASET));