private static boolean deleteWithTrash(URI uri, boolean useTrash){ Preconditions.checkArgument( URIBuilder.DATASET_SCHEME.equals(uri.getScheme()), "Not a dataset URI: " + uri); Pair<DatasetRepository, Map<String, String>> pair = Registration.lookupDatasetUri(URI.create(uri.getRawSchemeSpecificPart())); DatasetRepository repo = pair.first(); Map<String, String> uriOptions = pair.second(); return useTrash ? repo.moveToTrash(uriOptions.get(URIBuilder.NAMESPACE_OPTION), uriOptions.get(URIBuilder.DATASET_NAME_OPTION)) : repo.delete(uriOptions.get(URIBuilder.NAMESPACE_OPTION), uriOptions.get(URIBuilder.DATASET_NAME_OPTION)); }
private static void deleteJobDataset(JobContext jobContext) { DatasetRepository repo = getDatasetRepository(jobContext); repo.delete(TEMP_NAMESPACE, getJobDatasetName(jobContext)); }
private static void deleteJobDataset(JobContext jobContext) { DatasetRepository repo = getDatasetRepository(jobContext); repo.delete(TEMP_NAMESPACE, getJobDatasetName(jobContext)); }
@Override public void execute() throws MojoExecutionException, MojoFailureException { getConf(); // ensure properties are added to DefaultConfig if (uri != null) { Datasets.delete(uri); } else { LOG.warn( "kite.datasetName is deprecated, instead use kite.uri=<dataset-uri>"); Preconditions.checkArgument(datasetName != null, "kite.datasetName is required if kite.uri is not used"); DatasetRepository repo = getDatasetRepository(); repo.delete(datasetNamespace, datasetName); } } }
private static void deleteTaskAttemptDataset(TaskAttemptContext taskContext) { DatasetRepository repo = getDatasetRepository(taskContext); String taskAttemptDatasetName = getTaskAttemptDatasetName(taskContext); if (repo.exists(TEMP_NAMESPACE, taskAttemptDatasetName)) { repo.delete(TEMP_NAMESPACE, taskAttemptDatasetName); } }
@After public void removeStringsDataset() { repo.delete("ns", "strings"); }
private static void deleteTaskAttemptDataset(TaskAttemptContext taskContext) { DatasetRepository repo = getDatasetRepository(taskContext); String taskAttemptDatasetName = getTaskAttemptDatasetName(taskContext); if (repo.exists(TEMP_NAMESPACE, taskAttemptDatasetName)) { repo.delete(TEMP_NAMESPACE, taskAttemptDatasetName); } }
@After public void tearDown() { repo.delete("ns", "out"); }
@Override @SuppressWarnings("unchecked") public void commitTask(TaskAttemptContext taskContext) throws IOException { DatasetRepository repo = getDatasetRepository(taskContext); boolean inTempRepo = repo instanceof TemporaryDatasetRepository; Dataset<E> jobDataset = repo.load(TEMP_NAMESPACE, getJobDatasetName(taskContext)); String taskAttemptDatasetName = getTaskAttemptDatasetName(taskContext); if (repo.exists(TEMP_NAMESPACE, taskAttemptDatasetName)) { Dataset<E> taskAttemptDataset = repo.load(TEMP_NAMESPACE, taskAttemptDatasetName); ((Mergeable<Dataset<E>>) jobDataset).merge(taskAttemptDataset); if (!inTempRepo) { repo.delete(TEMP_NAMESPACE, taskAttemptDatasetName); } } }
@Override @SuppressWarnings("unchecked") public void commitTask(TaskAttemptContext taskContext) throws IOException { DatasetRepository repo = getDatasetRepository(taskContext); boolean inTempRepo = repo instanceof TemporaryDatasetRepository; Dataset<E> jobDataset = repo.load(TEMP_NAMESPACE, getJobDatasetName(taskContext)); String taskAttemptDatasetName = getTaskAttemptDatasetName(taskContext); if (repo.exists(TEMP_NAMESPACE, taskAttemptDatasetName)) { Dataset<E> taskAttemptDataset = repo.load(TEMP_NAMESPACE, taskAttemptDatasetName); ((Mergeable<Dataset<E>>) jobDataset).merge(taskAttemptDataset); if (!inTempRepo) { repo.delete(TEMP_NAMESPACE, taskAttemptDatasetName); } } }
@Test public void testDeleteStringUri() { URI datasetUri = new URIBuilder(repoUri, "ns", "test").build(); Datasets.delete(datasetUri.toString()); verify(repo).delete("ns", "test"); verifyNoMoreInteractions(repo); }
@Test public void testDelete() { URI datasetUri = new URIBuilder(repoUri, "ns", "test").build(); Datasets.delete(datasetUri); verify(repo).delete("ns", "test"); verifyNoMoreInteractions(repo); }
@Test public void testDeleteRemovesDatasetPath() throws IOException { ensureCreated(); Dataset<Record> dataset = repo.load(NAMESPACE, NAME); Path dataPath = new Path(dataset.getDescriptor().getLocation()); Assert.assertTrue(fileSystem.exists(dataPath)); repo.delete(NAMESPACE, NAME); Assert.assertFalse(fileSystem.exists(dataPath)); }
@Test public void testBasic() { DatasetRepository repo = DatasetRepositories.repositoryFor(repositoryUri); repo.delete("default", "test"); repo.create("default", "test", descriptor); RandomAccessDataset<Object> ds = Datasets .<Object, RandomAccessDataset<Object>>load(URI.create("dataset:hbase:" + zk + "/test"), Object.class); Assert.assertNotNull("Should load dataset", ds); Assert.assertTrue(ds instanceof DaoDataset); Assert.assertEquals("Descriptors should match", repo.load("default", "test").getDescriptor(), ds.getDescriptor()); repo.delete("default", "test"); }
@Before public void createStringsDataset() throws Exception { // create a dataset in HDFS repo.delete("ns", "strings"); repo.create("ns", "strings", new DatasetDescriptor.Builder() .schemaLiteral("\"string\"") .build()); }
@Test public void testCreateWithAllowNullSchema() { String name = "allowNull"; try { repo.create(NAMESPACE, name, new DatasetDescriptor.Builder() .schema(ReflectData.AllowNull.get().getSchema(ObjectPoJo.class)) .build()); } catch (RuntimeException e) { throw e; } finally { repo.delete(NAMESPACE, name); } }
@Test public void testAbsolute() { DatasetRepository repo = DatasetRepositories.repositoryFor("repo:file:/tmp/data"); repo.delete("ns", "test"); repo.create("ns", "test", descriptor); Dataset<Record> ds = Datasets.<Record, Dataset<Record>> load("dataset:file:/tmp/data/ns/test", Record.class); Assert.assertNotNull("Should load dataset", ds); Assert.assertTrue(ds instanceof FileSystemDataset); Assert.assertEquals("Locations should match", URI.create("file:/tmp/data/ns/test"), ds.getDescriptor().getLocation()); Assert.assertEquals("Descriptors should match", repo.load("ns", "test").getDescriptor(), ds.getDescriptor()); Assert.assertEquals("Should report correct namespace", "ns", ds.getNamespace()); Assert.assertEquals("Should report correct name", "test", ds.getName()); repo.delete("ns", "test"); }
@Test public void testAbsoluteRoot() { DatasetRepository repo = DatasetRepositories .repositoryFor("repo:hdfs://" + hdfsAuth + "/"); repo.delete("ns", "test"); repo.create("ns", "test", descriptor); Dataset<Object> ds = Datasets.<Object, Dataset<Object>> load("dataset:hdfs://" + hdfsAuth + "/ns/test", Object.class); Assert.assertNotNull("Should load dataset", ds); Assert.assertTrue(ds instanceof FileSystemDataset); Assert.assertEquals("Locations should match", URI.create("hdfs://" + hdfsAuth + "/ns/test"), ds.getDescriptor().getLocation()); Assert.assertEquals("Descriptors should match", repo.load("ns", "test").getDescriptor(), ds.getDescriptor()); Assert.assertEquals("Should report correct namespace", "ns", ds.getNamespace()); Assert.assertEquals("Should report correct name", "test", ds.getName()); repo.delete("ns", "test"); }
@Test public void testExists() { Assert.assertFalse(repo.exists(NAMESPACE, "test1")); repo.create(NAMESPACE, "test1", new DatasetDescriptor.Builder() .schema(testSchema).build()); Assert.assertTrue(repo.exists(NAMESPACE, "test1")); repo.delete(NAMESPACE, "test1"); Assert.assertFalse(repo.exists(NAMESPACE, "test1")); }