@Override public void upgrade(UpgradeContext context) throws Exception { final NamespaceService namespaceService = new NamespaceServiceImpl(context.getKVStoreProvider()); List<SourceConfig> sources = namespaceService.getSources(); for (SourceConfig sourceConfig : sources) { // Pre-1.5, the config object for internal sources was null. Deleting the internal sources without a config will // ensure that they get recreated on startup. if (sourceConfig.getConfig() == null) { System.out.printf(" deleting '%s'%n", sourceConfig.getName()); // following may throw an exception, we let it propagate to fail the upgrade namespaceService.deleteSource(sourceConfig.getKey(), sourceConfig.getTag()); } } }
private void assertNoDatasetsAfterSourceDeletion() throws Exception { final SourceConfig sourceConfig = namespaceService.getSource(mockUpKey); namespaceService.deleteSource(mockUpKey, sourceConfig.getTag()); assertEquals(0, Iterables.size(namespaceService.getAllDatasets(mockUpKey))); }
@Test public void testByPath() throws Exception { Source createdSource = createSource(); // test getting a source by name Source source = expectSuccess(getBuilder(getPublicAPI(3).path(CATALOG_PATH).path("by-path").path(createdSource.getName())).buildGet(), new GenericType<Source>() {}); assertEquals(source.getId(), createdSource.getId()); // test getting a folder by path expectSuccess(getBuilder(getPublicAPI(3).path(CATALOG_PATH).path("by-path").path(createdSource.getName()).path("json")).buildGet(), new GenericType<Folder>() {}); // test getting a file with a url character in name (?) expectSuccess(getBuilder(getPublicAPI(3).path(CATALOG_PATH).path("by-path").path(createdSource.getName()).path("testfiles").path("file_with_?.json")).buildGet(), new GenericType<File>() {}); newNamespaceService().deleteSource(new NamespaceKey(source.getName()), source.getTag()); }
@After public void deleteSource() throws NamespaceException { newNamespaceService() .deleteSource(new NamespaceKey(source.getName()), source.getTag()); }
public void deleteSource() { NamespaceKey key = new SourcePath(TEST_SOURCE).toNamespaceKey(); SourceConfig config; try { config = getNamespaceService().getSource(key); if (config != null) { getNamespaceService().deleteSource(key, config.getTag()); } } catch (NamespaceException e) { } }
@Test public void testSourceBrowsing() throws Exception { Source source = createSource(); // browse to the json directory String id = getFolderIdByName(source.getChildren(), "\"json\""); assertNotNull(id, "Failed to find json directory"); // deleting a folder on a source should fail expectStatus(Response.Status.BAD_REQUEST, getBuilder(getPublicAPI(3).path(CATALOG_PATH).path(com.dremio.common.utils.PathUtils.encodeURIComponent(id))).buildDelete()); newNamespaceService().deleteSource(new NamespaceKey(source.getName()), source.getTag()); }
namespaceService.deleteSource(config.getKey(), config.getTag()); sourceDataStore.delete(config.getKey()); namespaceService.deleteSource(config.getKey(), config.getTag()); sourceDataStore.delete(config.getKey());
@Test public void testSourceHasDefaultTTL() throws Exception { final String sourceName = "src2"; final NASConf nas = new NASConf(); nas.path = folder.getRoot().getPath(); SourceUI source = new SourceUI(); source.setName(sourceName); source.setCtime(System.currentTimeMillis()); source.setConfig(nas); expectSuccess( getBuilder(getAPIv2().path(String.format("/source/%s", sourceName))) .buildPut(Entity.entity(source, JSON))); final SourceUI result = expectSuccess( getBuilder(getAPIv2().path(String.format("/source/%s", sourceName))).buildGet(), SourceUI.class ); assertEquals(source.getFullPathList(), result.getFullPathList()); assertNotNull(result.getAccelerationRefreshPeriod()); assertNotNull(result.getAccelerationGracePeriod()); newNamespaceService().deleteSource(new SourcePath(sourceName).toNamespaceKey(), result.getTag()); }
@Test public void testUpdateSourceErrors() throws Exception { SourceConfig sourceConfig = new SourceConfig(); sourceConfig.setName("Foopy5"); NASConf nasConfig = new NASConf(); sourceConfig.setType(nasConfig.getType()); nasConfig.path = "/"; sourceConfig.setConfig(nasConfig.toBytesString()); SourceConfig createdSourceConfig = newSourceService().registerSourceWithRuntime(sourceConfig); final AccelerationSettings settings = new AccelerationSettings() .setMethod(RefreshMethod.FULL) .setRefreshPeriod(TimeUnit.HOURS.toMillis(2)) .setGracePeriod(TimeUnit.HOURS.toMillis(6)); SourceResource.SourceDeprecated updatedSource = new SourceResource.SourceDeprecated(createdSourceConfig, settings, reader); // test updating non-existent source expectStatus(Response.Status.NOT_FOUND, getBuilder(getPublicAPI(3).path(SOURCES_PATH).path("badid")).buildPut(Entity.entity(updatedSource, JSON))); // test wrong tag updatedSource.setTag("badtag"); expectStatus(Response.Status.CONFLICT, getBuilder(getPublicAPI(3).path(SOURCES_PATH).path(createdSourceConfig.getId().getId())).buildPut(Entity.entity(updatedSource, JSON))); newNamespaceService().deleteSource(new SourcePath(updatedSource.getName()).toNamespaceKey(), createdSourceConfig.getTag()); }
@Test public void testAddSourceWithAccelerationTTL() throws Exception { final String sourceName = "src"; final long refreshPeriod = TimeUnit.HOURS.toMillis(4); final long gracePeriod = TimeUnit.HOURS.toMillis(12); { final NASConf nas = new NASConf(); nas.path = folder.getRoot().getPath(); SourceUI source = new SourceUI(); source.setName(sourceName); source.setCtime(System.currentTimeMillis()); source.setAccelerationRefreshPeriod(refreshPeriod); source.setAccelerationGracePeriod(gracePeriod); source.setConfig(nas); expectSuccess( getBuilder(getAPIv2().path(String.format("/source/%s", sourceName))) .buildPut(Entity.entity(source, JSON))); final SourceUI result = expectSuccess( getBuilder(getAPIv2().path(String.format("/source/%s", sourceName))).buildGet(), SourceUI.class ); assertEquals(source.getFullPathList(), result.getFullPathList()); assertEquals(source.getAccelerationRefreshPeriod(), result.getAccelerationRefreshPeriod()); assertEquals(source.getAccelerationGracePeriod(), result.getAccelerationGracePeriod()); newNamespaceService().deleteSource(new SourcePath(sourceName).toNamespaceKey(), result.getTag()); } }
@Test public void testAddSource() throws Exception { SourceResource.SourceDeprecated newSource = new SourceResource.SourceDeprecated(); newSource.setName("Foopy"); newSource.setType("NAS"); NASConf config = new NASConf(); config.path = "/"; newSource.setConfig(config); SourceResource.SourceDeprecated source = expectSuccess(getBuilder(getPublicAPI(3).path(SOURCES_PATH)).buildPost(Entity.entity(newSource, JSON)), SourceResource.SourceDeprecated.class); assertEquals(source.getName(), newSource.getName()); assertNotNull(source.getState()); newNamespaceService().deleteSource(new SourcePath(source.getName()).toNamespaceKey(), source.getTag()); }
@Test public void testSourceHasDefaultRefreshPolicy() throws Exception { final String sourceName = "src3"; final NASConf nas = new NASConf(); nas.path = folder.getRoot().getPath() ; SourceUI source = new SourceUI(); source.setName(sourceName); source.setCtime(System.currentTimeMillis()); source.setConfig(nas); expectSuccess( getBuilder(getAPIv2().path(String.format("/source/%s", sourceName))) .buildPut(Entity.entity(source, JSON))); final SourceUI result = expectSuccess( getBuilder(getAPIv2().path(String.format("/source/%s", sourceName))).buildGet(), SourceUI.class ); assertEquals(source.getFullPathList(), result.getFullPathList()); assertNotNull(source.getMetadataPolicy()); assertEquals(CatalogService.DEFAULT_AUTHTTLS_MILLIS, result.getMetadataPolicy().getAuthTTLMillis()); assertEquals(CatalogService.DEFAULT_REFRESH_MILLIS, result.getMetadataPolicy().getNamesRefreshMillis()); assertEquals(CatalogService.DEFAULT_REFRESH_MILLIS, result.getMetadataPolicy().getDatasetDefinitionRefreshAfterMillis()); assertEquals(CatalogService.DEFAULT_EXPIRE_MILLIS, result.getMetadataPolicy().getDatasetDefinitionExpireAfterMillis()); newNamespaceService().deleteSource(new SourcePath(sourceName).toNamespaceKey(), result.getTag()); } }
newNamespaceService().deleteSource(new NamespaceKey(source.getName()), source.getTag());
@Test public void testUpdateSource() throws Exception { SourceConfig sourceConfig = new SourceConfig(); sourceConfig.setName("Foopy2"); NASConf nasConfig = new NASConf(); sourceConfig.setType(nasConfig.getType()); nasConfig.path = "/"; sourceConfig.setConfig(nasConfig.toBytesString()); SourceConfig createdSourceConfig = newSourceService().registerSourceWithRuntime(sourceConfig); final AccelerationSettings settings = new AccelerationSettings() .setMethod(RefreshMethod.FULL) .setRefreshPeriod(TimeUnit.HOURS.toMillis(2)) .setGracePeriod(TimeUnit.HOURS.toMillis(6)); SourceResource.SourceDeprecated updatedSource = new SourceResource.SourceDeprecated(createdSourceConfig, settings, reader); updatedSource.setDescription("Desc"); SourceResource.SourceDeprecated source = expectSuccess(getBuilder(getPublicAPI(3).path(SOURCES_PATH).path(createdSourceConfig.getId().getId())).buildPut(Entity.entity(updatedSource, JSON)), SourceResource.SourceDeprecated.class); assertEquals(source.getDescription(), "Desc"); assertEquals(source.getTag(), "1"); assertNotNull(source.getState()); newNamespaceService().deleteSource(new SourcePath(source.getName()).toNamespaceKey(), source.getTag()); }
newNamespaceService().deleteSource(new SourcePath(createdSourceConfig.getName()).toNamespaceKey(), "1");
@Test public void testSourceEditWithoutSecret() throws Exception { // fakesource only works if password is the same as the name, else with fail to create final FakeSource fakeConf = new FakeSource(); fakeConf.password = "fake"; fakeConf.isAwesome = true; final Source newSource = new Source(); newSource.setName("fake"); newSource.setType("FAKESOURCE"); newSource.setConfig(fakeConf); // create the source Source source = expectSuccess(getBuilder(getPublicAPI(3).path(CATALOG_PATH)).buildPost(Entity.json(newSource)), new GenericType<Source>() {}); FakeSource config = (FakeSource) source.getConfig(); // we should get back the use existing secret const assertTrue(config.isAwesome); assertEquals(config.password, ConnectionConf.USE_EXISTING_SECRET_VALUE); // verify that saving with the const works config.isAwesome = false; source.setConfig(config); source = expectSuccess(getBuilder(getPublicAPI(3).path(CATALOG_PATH).path(source.getId())).buildPut(Entity.json(source)), new GenericType<Source>() {}); config = (FakeSource) source.getConfig(); assertEquals(source.getTag(), "1"); assertFalse(config.isAwesome); newNamespaceService().deleteSource(new NamespaceKey(source.getName()), source.getTag()); }
@Test public void testUpdateSourceBoundaryValues() throws Exception { SourceConfig sourceConfig = new SourceConfig(); sourceConfig.setName("Foopy2"); NASConf nasConfig = new NASConf(); sourceConfig.setType(nasConfig.getType()); nasConfig.path = "/"; sourceConfig.setConfig(nasConfig.toBytesString()); SourceConfig createdSourceConfig = newSourceService().registerSourceWithRuntime(sourceConfig); final AccelerationSettings settings = new AccelerationSettings() .setMethod(RefreshMethod.FULL) .setRefreshPeriod(TimeUnit.HOURS.toMillis(2)) .setGracePeriod(TimeUnit.HOURS.toMillis(6)); SourceResource.SourceDeprecated updatedSource = new SourceResource.SourceDeprecated(createdSourceConfig, settings, reader); updatedSource.getMetadataPolicy().setDatasetRefreshAfterMs(MetadataPolicy.ONE_MINUTE_IN_MS); updatedSource.getMetadataPolicy().setAuthTTLMs(MetadataPolicy.ONE_MINUTE_IN_MS); updatedSource.getMetadataPolicy().setNamesRefreshMs(MetadataPolicy.ONE_MINUTE_IN_MS); SourceResource.SourceDeprecated source = expectSuccess(getBuilder(getPublicAPI(3).path(SOURCES_PATH).path(createdSourceConfig.getId().getId())).buildPut(Entity.entity(updatedSource, JSON)), SourceResource.SourceDeprecated.class); assertEquals(source.getMetadataPolicy().getAuthTTLMs(), updatedSource.getMetadataPolicy().getAuthTTLMs()); assertEquals(source.getMetadataPolicy().getDatasetRefreshAfterMs(), updatedSource.getMetadataPolicy().getDatasetRefreshAfterMs()); assertEquals(source.getMetadataPolicy().getNamesRefreshMs(), updatedSource.getMetadataPolicy().getNamesRefreshMs()); newNamespaceService().deleteSource(new SourcePath(createdSourceConfig.getName()).toNamespaceKey(), source.getTag()); }
namespaceService.deleteSource(new NamespaceKey("src2"), "1234"); fail("deleteSource didn't throw exception"); } catch (ConcurrentModificationException nfe) { namespaceService.deleteSource(new NamespaceKey("src1"), newSrc1.getTag()); verifyDSNotInNamespace(namespaceService, new NamespaceKey("src1.fld1.ds2")); namespaceService.deleteSource(new NamespaceKey("src2"), newSrc2.getTag()); verifySourceNotInNamespace(namespaceService, new NamespaceKey("src2"));