public FileSystemDatasetRepository(Configuration conf, Path rootDirectory) { this(conf, rootDirectory, new FileSystemMetadataProvider(conf, rootDirectory)); }
public static void createMetadata(FileSystem fs, Path path, String name) throws IOException { fs.mkdirs(path); FileSystemMetadataProvider.writeDescriptor(fs, path, name, descriptor); }
Path expectedPath = pathForMetadata(namespace, name); if (DEFAULT_NAMESPACE.equals(namespace)) { checkExists(rootFileSystem, expectedPath); return expectedPath; } catch (DatasetNotFoundException e) { Path backwardCompatiblePath = new Path(rootDirectory, new Path( name.replace('.', Path.SEPARATOR_CHAR), METADATA_DIRECTORY)); checkExists(rootFileSystem, backwardCompatiblePath); return backwardCompatiblePath; } catch (DatasetNotFoundException _) { checkExists(rootFileSystem, expectedPath); return expectedPath;
Path metadataPath = find(namespace, name); checkExists(getFileSytem(), schemaPath); location = expectedPathForDataset(namespace, name);
@Test public void testHdfsAbsolute() throws URISyntaxException { URI hdfsUri = getDFS().getUri(); URI repositoryUri = new URI("repo:hdfs://" + hdfsUri.getAuthority() + "/tmp/dsr-repo-test"); DatasetRepository repository = DatasetRepositories.repositoryFor(repositoryUri); // We only do the deeper implementation checks one per combination. Assert.assertNotNull("Received a repository", repository); Assert.assertTrue("Repo is a FileSystem repo", repository instanceof FileSystemDatasetRepository); MetadataProvider provider = ((FileSystemDatasetRepository) repository) .getMetadataProvider(); Assert.assertTrue("Repo is using a FileSystemMetadataProvider", provider instanceof FileSystemMetadataProvider); FileSystemMetadataProvider fsProvider = (FileSystemMetadataProvider) provider; Assert.assertTrue("FileSystem is a DistributedFileSystem", fsProvider.getFileSytem() instanceof DistributedFileSystem); Path expected = fsProvider.getFileSytem().makeQualified( new Path("/tmp/dsr-repo-test")); Assert.assertEquals("Root directory should be the correct qualified path", expected, fsProvider.getRootDirectory()); Assert.assertEquals("Repository URI", repositoryUri, repository.getUri()); }
@Override public DatasetDescriptor update(String namespace, String name, DatasetDescriptor descriptor) { Preconditions.checkNotNull(namespace, "Namespace cannot be null"); Preconditions.checkNotNull(name, "Dataset name cannot be null"); Preconditions.checkNotNull(descriptor, "Descriptor cannot be null"); Compatibility.check(namespace, name, descriptor); LOG.debug("Saving dataset metadata name: {} descriptor: {}", name, descriptor); Path metadataPath = find(namespace, name); writeDescriptor(rootFileSystem, metadataPath, name, descriptor); return descriptor; }
@Override public DatasetDescriptor create(String namespace, String name, DatasetDescriptor descriptor) { Preconditions.checkNotNull(namespace, "Namespace cannot be null"); Preconditions.checkNotNull(name, "Dataset name cannot be null"); Preconditions.checkNotNull(descriptor, "Descriptor cannot be null"); Compatibility.check(namespace, name, descriptor); LOG.debug("Saving dataset metadata name:{} descriptor:{}", name, descriptor); // no need to check backward-compatibility when creating new datasets Path metadataLocation = pathForMetadata(namespace, name); try { if (rootFileSystem.exists(metadataLocation)) { throw new DatasetExistsException( "Descriptor directory already exists: " + metadataLocation); } // create the directory so that update can do the rest of the work rootFileSystem.mkdirs(metadataLocation); } catch (IOException e) { throw new DatasetIOException( "Unable to create metadata directory: " + metadataLocation + " for dataset: " + name, e); } writeDescriptor(rootFileSystem, metadataLocation, name, descriptor); return descriptor; }
@SuppressWarnings("deprecation") @Override public Set<String> namespaces() { Set<String> namespaces = Sets.newHashSet(); try { FileStatus[] entries = rootFileSystem.listStatus(rootDirectory, PathFilters.notHidden()); for (FileStatus entry : entries) { if (entry.isDir()) { // may want to add a check: !RESERVED_NAMES.contains(name) if (isNamespace(entry.getPath())) { namespaces.add(entry.getPath().getName()); } else if (isDataset(entry.getPath())) { // add the default namespace for datasets with no namespace namespaces.add(DEFAULT_NAMESPACE); } } } } catch (FileNotFoundException ex) { // the repo hasn't created any files yet return namespaces; } catch (IOException ex) { throw new DatasetIOException("Could not list namespaces", ex); } return namespaces; }
@Override public boolean moveToTrash(String namespace, String name) { return deleteWithTrash(namespace, name, true); }
@Override public boolean exists(String namespace, String name) { Preconditions.checkNotNull(namespace, "Namespace cannot be null"); Preconditions.checkNotNull(name, "Dataset name cannot be null"); try { find(namespace, name); return true; } catch (DatasetNotFoundException e) { return false; } }
/** * Returns whether the given {@code Path} contains directories with * {@code Dataset} metadata. * * @param dir a Path to check * @return {@code true} if there is a direct sub-directory with metadata * @throws IOException */ @SuppressWarnings("deprecation") private boolean isNamespace(Path dir) throws IOException { FileStatus[] stats = rootFileSystem.listStatus(dir, PathFilters.notHidden()); for (FileStatus stat : stats) { if (stat.isDir() && isDataset(stat.getPath())) { return true; } } return false; }
DatasetDescriptor descriptor) { checkExists(fs, metadataLocation);
@Test public void testLocalAbsolute() throws URISyntaxException { URI repositoryUri = new URI("repo:file:///tmp/dsr-repo-test"); DatasetRepository repository = DatasetRepositories.repositoryFor(repositoryUri); FileSystemMetadataProvider provider = (FileSystemMetadataProvider) ((FileSystemDatasetRepository) repository).getMetadataProvider(); Assert.assertEquals("Root directory should be the correct qualified path", new Path("file:/tmp/dsr-repo-test"), provider.getRootDirectory()); Assert.assertEquals("Repository URI", repositoryUri, repository.getUri()); }
@Test public void testLocalRelative() throws URISyntaxException { URI repositoryUri = new URI("repo:file:target/dsr-repo-test"); DatasetRepository repository = DatasetRepositories.repositoryFor(repositoryUri); // We only do the deeper implementation checks one per combination. Assert.assertNotNull("Received a repository", repository); Assert.assertTrue("Repo is a FileSystem repo", repository instanceof FileSystemDatasetRepository); MetadataProvider provider = ((FileSystemDatasetRepository) repository) .getMetadataProvider(); Assert.assertTrue("Repo is using a FileSystemMetadataProvider", provider instanceof FileSystemMetadataProvider); FileSystemMetadataProvider fsProvider = (FileSystemMetadataProvider) provider; Assert.assertTrue("FileSystem is a LocalFileSystem", fsProvider.getFileSytem() instanceof LocalFileSystem); Path expected = fsProvider.getFileSytem().makeQualified( new Path("target/dsr-repo-test")); Assert.assertEquals("Root directory should be the correct qualified path", expected, fsProvider.getRootDirectory()); Assert.assertEquals("Repository URI scheme", "repo", repository.getUri() .getScheme()); Assert.assertEquals("Repository URI scheme", expected.toUri(), new URI(repository.getUri().getSchemeSpecificPart())); }
@Override public boolean delete(String namespace, String name) { return deleteWithTrash(namespace, name, false); }
metadataDirectory = find(namespace, name); } catch (DatasetNotFoundException _) { return false;
@Override public MetadataProvider newProvider(Configuration conf) { return new FileSystemMetadataProvider(conf, testDirectory); }
@Override public MetadataProvider newProvider(Configuration conf) { this.testDirectory = new Path(Files.createTempDir().getAbsolutePath()); return new FileSystemMetadataProvider(conf, testDirectory); }
@BeforeClass public static void createMixedLayoutRepository() throws IOException { local = FileSystem.get(conf); local.delete(root, true); // clean any old data createMetadata( local, new Path(root, new Path("old_1", ".metadata")), "old_1"); createMetadata( local, new Path(root, new Path("old_2", ".metadata")), "old_2"); provider = new FileSystemMetadataProvider(conf, root); provider.create("ns", "new_1", descriptor); provider.create("ns2", "new_2", descriptor); }