source.setName("LocalFS1"); source.setCtime(System.currentTimeMillis()); final NASConf nas = new NASConf(); nas.path = path.toFile().getPath(); source.setConfig(nas); source.setName("LocalFS2"); source.setCtime(System.currentTimeMillis()); final NASConf nas = new NASConf(); nas.path = path.toFile().getPath(); source.setConfig(nas); source.setName("LocalRootRO"); source.setCtime(System.currentTimeMillis()); final NASConf nas = new NASConf(); nas.path = "/"; source.setConfig(nas);
@Test public void testDeleteSource() throws Exception { SourceConfig sourceConfig = new SourceConfig(); sourceConfig.setName("Foopy3"); NASConf nasConfig = new NASConf(); sourceConfig.setType(nasConfig.getType()); nasConfig.path = "/"; sourceConfig.setConfig(nasConfig.toBytesString()); SourceConfig createdSourceConfig = newSourceService().registerSourceWithRuntime(sourceConfig); expectSuccess(getBuilder(getPublicAPI(3).path(SOURCES_PATH).path(createdSourceConfig.getId().getId())).buildDelete()); }
public static SourceConfig addSource(NamespaceService ns, String name) throws Exception { final NASConf conf = new NASConf(); conf.path = Files.createTempDirectory(null).toString(); final SourceConfig src = new SourceConfig() .setName(name) .setCtime(100L) .setConnectionConf(conf) .setAccelerationRefreshPeriod(TimeUnit.HOURS.toMillis(24)) .setAccelerationGracePeriod(TimeUnit.HOURS.toMillis(48)); try { l(CatalogService.class).createSourceIfMissingWithThrow(src); } catch (ConcurrentModificationException e) { // noop - changed signature to throw } return src; }
@Test public void testGetSource() throws Exception { SourceConfig sourceConfig = new SourceConfig(); sourceConfig.setName("Foopy4"); NASConf nasConfig = new NASConf(); sourceConfig.setType(nasConfig.getType()); nasConfig.path = "/"; sourceConfig.setConfig(nasConfig.toBytesString()); SourceConfig createdSourceConfig = newSourceService().registerSourceWithRuntime(sourceConfig); SourceResource.SourceDeprecated source = expectSuccess(getBuilder(getPublicAPI(3).path(SOURCES_PATH).path(createdSourceConfig.getId().getId())).buildGet(), SourceResource.SourceDeprecated.class); assertEquals(source.getName(), sourceConfig.getName()); assertNotNull(source.getState()); }
@Before public void addWritableSource() throws IOException { NASConf writable = new NASConf(); writable.allowCreateDrop = true; File folderWritable = folder.newFolder(); // add an existing folder. new File(folderWritable, "existing").mkdirs(); writable.path = folderWritable.getCanonicalPath(); SourceConfig conf = new SourceConfig(); conf.setConnectionConf(writable); conf.setMetadataPolicy(CatalogService.DEFAULT_METADATA_POLICY_WITH_AUTO_PROMOTE); conf.setName("writable"); getSabotContext().getCatalogService().createSourceIfMissingWithThrow(conf); }
@Test public void testUpdateSourceErrors() throws Exception { SourceConfig sourceConfig = new SourceConfig(); sourceConfig.setName("Foopy5"); NASConf nasConfig = new NASConf(); sourceConfig.setType(nasConfig.getType()); nasConfig.path = "/"; sourceConfig.setConfig(nasConfig.toBytesString()); SourceConfig createdSourceConfig = newSourceService().registerSourceWithRuntime(sourceConfig); final AccelerationSettings settings = new AccelerationSettings() .setMethod(RefreshMethod.FULL) .setRefreshPeriod(TimeUnit.HOURS.toMillis(2)) .setGracePeriod(TimeUnit.HOURS.toMillis(6)); SourceResource.SourceDeprecated updatedSource = new SourceResource.SourceDeprecated(createdSourceConfig, settings, reader); // test updating non-existent source expectStatus(Response.Status.NOT_FOUND, getBuilder(getPublicAPI(3).path(SOURCES_PATH).path("badid")).buildPut(Entity.entity(updatedSource, JSON))); // test wrong tag updatedSource.setTag("badtag"); expectStatus(Response.Status.CONFLICT, getBuilder(getPublicAPI(3).path(SOURCES_PATH).path(createdSourceConfig.getId().getId())).buildPut(Entity.entity(updatedSource, JSON))); newNamespaceService().deleteSource(new SourcePath(updatedSource.getName()).toNamespaceKey(), createdSourceConfig.getTag()); }
@Test public void testInvalidSource() throws Exception { SourceUI sourceUI = new SourceUI(); sourceUI.setName("A.B"); NASConf sourceConfig = new NASConf(); sourceConfig.path = "/"; sourceUI.setConfig(sourceConfig); expectError(CLIENT_ERROR, getBuilder(getAPIv2().path("source/A.B")).buildPut(Entity.json(sourceUI)), ValidationErrorMessage.class); }
@Test public void testUpdateSource() throws Exception { SourceConfig sourceConfig = new SourceConfig(); sourceConfig.setName("Foopy2"); NASConf nasConfig = new NASConf(); sourceConfig.setType(nasConfig.getType()); nasConfig.path = "/"; sourceConfig.setConfig(nasConfig.toBytesString()); SourceConfig createdSourceConfig = newSourceService().registerSourceWithRuntime(sourceConfig); final AccelerationSettings settings = new AccelerationSettings() .setMethod(RefreshMethod.FULL) .setRefreshPeriod(TimeUnit.HOURS.toMillis(2)) .setGracePeriod(TimeUnit.HOURS.toMillis(6)); SourceResource.SourceDeprecated updatedSource = new SourceResource.SourceDeprecated(createdSourceConfig, settings, reader); updatedSource.setDescription("Desc"); SourceResource.SourceDeprecated source = expectSuccess(getBuilder(getPublicAPI(3).path(SOURCES_PATH).path(createdSourceConfig.getId().getId())).buildPut(Entity.entity(updatedSource, JSON)), SourceResource.SourceDeprecated.class); assertEquals(source.getDescription(), "Desc"); assertEquals(source.getTag(), "1"); assertNotNull(source.getState()); newNamespaceService().deleteSource(new SourcePath(source.getName()).toNamespaceKey(), source.getTag()); }
final NASConf nas = new NASConf(); nas.path = System.getProperty("user.dir") + "/src/test/resources/datasets"; sourceUI.setConfig(nas);
sourceConfig.setName("Foopy2"); NASConf nasConfig = new NASConf(); sourceConfig.setType(nasConfig.getType()); nasConfig.path = "/"; sourceConfig.setConfig(nasConfig.toBytesString());
@Before public void addNonWritableSource() throws IOException { NASConf writable = new NASConf(); writable.allowCreateDrop = false; File folderWritable = folder.newFolder(); writable.path = folderWritable.getCanonicalPath(); SourceConfig conf = new SourceConfig(); conf.setConnectionConf(writable); conf.setMetadataPolicy(CatalogService.DEFAULT_METADATA_POLICY_WITH_AUTO_PROMOTE); conf.setName("nonwritable"); getSabotContext().getCatalogService().createSourceIfMissingWithThrow(conf); }
@Test public void testUpdateSourceBoundaryValues() throws Exception { SourceConfig sourceConfig = new SourceConfig(); sourceConfig.setName("Foopy2"); NASConf nasConfig = new NASConf(); sourceConfig.setType(nasConfig.getType()); nasConfig.path = "/"; sourceConfig.setConfig(nasConfig.toBytesString()); SourceConfig createdSourceConfig = newSourceService().registerSourceWithRuntime(sourceConfig); final AccelerationSettings settings = new AccelerationSettings() .setMethod(RefreshMethod.FULL) .setRefreshPeriod(TimeUnit.HOURS.toMillis(2)) .setGracePeriod(TimeUnit.HOURS.toMillis(6)); SourceResource.SourceDeprecated updatedSource = new SourceResource.SourceDeprecated(createdSourceConfig, settings, reader); updatedSource.getMetadataPolicy().setDatasetRefreshAfterMs(MetadataPolicy.ONE_MINUTE_IN_MS); updatedSource.getMetadataPolicy().setAuthTTLMs(MetadataPolicy.ONE_MINUTE_IN_MS); updatedSource.getMetadataPolicy().setNamesRefreshMs(MetadataPolicy.ONE_MINUTE_IN_MS); SourceResource.SourceDeprecated source = expectSuccess(getBuilder(getPublicAPI(3).path(SOURCES_PATH).path(createdSourceConfig.getId().getId())).buildPut(Entity.entity(updatedSource, JSON)), SourceResource.SourceDeprecated.class); assertEquals(source.getMetadataPolicy().getAuthTTLMs(), updatedSource.getMetadataPolicy().getAuthTTLMs()); assertEquals(source.getMetadataPolicy().getDatasetRefreshAfterMs(), updatedSource.getMetadataPolicy().getDatasetRefreshAfterMs()); assertEquals(source.getMetadataPolicy().getNamesRefreshMs(), updatedSource.getMetadataPolicy().getNamesRefreshMs()); newNamespaceService().deleteSource(new SourcePath(createdSourceConfig.getName()).toNamespaceKey(), source.getTag()); }
@Test public void testValidSource() throws Exception { SourceUI sourceUI = new SourceUI(); sourceUI.setName("AB"); NASConf sourceConfig = new NASConf(); sourceConfig.path = "/"; sourceUI.setConfig(sourceConfig); expectSuccess(getBuilder(getAPIv2().path("source/AB")).buildPut(Entity.json(sourceUI))); }
private Source createSource() { NASConf nasConf = new NASConf(); nasConf.path = TestTools.getWorkingPath() + "/src/test/resources"; Source newSource = new Source(); newSource.setName("catalog-test"); newSource.setType("NAS"); newSource.setConfig(nasConf); // create the source return expectSuccess(getBuilder(getPublicAPI(3).path(CATALOG_PATH)).buildPost(Entity.json(newSource)), new GenericType<Source>() {}); }
@Test public void previewDataForPhysicalDataset() throws Exception { final NASConf nas = new NASConf(); nas.path = new File("src/test/resources/datasets").getAbsolutePath(); SourceUI source = new SourceUI(); source.setName("testNAS"); source.setConfig(nas); source.setMetadataPolicy(UIMetadataPolicy.of(CatalogService.DEFAULT_METADATA_POLICY_WITH_AUTO_PROMOTE)); final SourceService sourceService = newSourceService(); sourceService.registerSourceWithRuntime(source); InitialDataPreviewResponse resp = getPreview(new DatasetPath(asList("testNAS", "users.json"))); assertEquals(3, resp.getData().getReturnedRowCount()); assertEquals(2, resp.getData().getColumns().size()); JobDataFragment data1 = getData(resp.getPaginationUrl(), 0, 2); assertEquals(2, data1.getReturnedRowCount()); assertEquals(2, data1.getColumns().size()); }
@Before public void setup() throws Exception { final NASConf nas = new NASConf(); nas.path = folder.getRoot().getPath(); SourceUI source = new SourceUI(); source.setName(SOURCE_NAME); source.setCtime(System.currentTimeMillis()); source.setAccelerationRefreshPeriod(DEFAULT_REFRESH_PERIOD); source.setAccelerationGracePeriod(DEFAULT_GRACE_PERIOD); // Please note: if this source is ever refreshed, the physical dataset added below will disappear source.setMetadataPolicy(UIMetadataPolicy.of(CatalogService.NEVER_REFRESH_POLICY)); source.setConfig(nas); getSourceService().registerSourceWithRuntime(source); addPhysicalDataset(DATASET_PATH, DatasetType.PHYSICAL_DATASET); }
@Test public void testSourceHasDefaultTTL() throws Exception { final String sourceName = "src2"; final NASConf nas = new NASConf(); nas.path = folder.getRoot().getPath(); SourceUI source = new SourceUI(); source.setName(sourceName); source.setCtime(System.currentTimeMillis()); source.setConfig(nas); expectSuccess( getBuilder(getAPIv2().path(String.format("/source/%s", sourceName))) .buildPut(Entity.entity(source, JSON))); final SourceUI result = expectSuccess( getBuilder(getAPIv2().path(String.format("/source/%s", sourceName))).buildGet(), SourceUI.class ); assertEquals(source.getFullPathList(), result.getFullPathList()); assertNotNull(result.getAccelerationRefreshPeriod()); assertNotNull(result.getAccelerationGracePeriod()); newNamespaceService().deleteSource(new SourcePath(sourceName).toNamespaceKey(), result.getTag()); }
@Test public void testAddSourceWithAccelerationTTL() throws Exception { final String sourceName = "src"; final long refreshPeriod = TimeUnit.HOURS.toMillis(4); final long gracePeriod = TimeUnit.HOURS.toMillis(12); { final NASConf nas = new NASConf(); nas.path = folder.getRoot().getPath(); SourceUI source = new SourceUI(); source.setName(sourceName); source.setCtime(System.currentTimeMillis()); source.setAccelerationRefreshPeriod(refreshPeriod); source.setAccelerationGracePeriod(gracePeriod); source.setConfig(nas); expectSuccess( getBuilder(getAPIv2().path(String.format("/source/%s", sourceName))) .buildPut(Entity.entity(source, JSON))); final SourceUI result = expectSuccess( getBuilder(getAPIv2().path(String.format("/source/%s", sourceName))).buildGet(), SourceUI.class ); assertEquals(source.getFullPathList(), result.getFullPathList()); assertEquals(source.getAccelerationRefreshPeriod(), result.getAccelerationRefreshPeriod()); assertEquals(source.getAccelerationGracePeriod(), result.getAccelerationGracePeriod()); newNamespaceService().deleteSource(new SourcePath(sourceName).toNamespaceKey(), result.getTag()); } }
final NASConf nas = new NASConf(); nas.path = new File("src/test/resources").getAbsolutePath(); SourceUI source = new SourceUI();
@Test public void previewTableInSubschemaExposedAsTopLevelSchema() throws Exception { // Namespace is cleaned up before the test. So make sure to create a home directory for the test user. final NamespaceKey homeKey = new HomePath(HomeName.getUserHomePath(DEFAULT_USERNAME)).toNamespaceKey(); final NamespaceService ns = l(NamespaceService.class); if (!ns.exists(homeKey, Type.HOME)) { final HomeConfig homeConfig = new HomeConfig().setOwner(DEFAULT_USERNAME); ns.addOrUpdateHome(homeKey, homeConfig); } final NASConf nas = new NASConf(); nas.path = new File("src/test/resources").getAbsolutePath(); SourceUI source = new SourceUI(); source.setName("testNAS"); source.setConfig(nas); source.setMetadataPolicy(UIMetadataPolicy.of(CatalogService.DEFAULT_METADATA_POLICY_WITH_AUTO_PROMOTE)); final SourceService sourceService = newSourceService(); sourceService.registerSourceWithRuntime(source); InitialDataPreviewResponse resp = getPreview(new DatasetPath(asList("testNAS", "datasets", "users.json"))); // Now try to query the table as "testNAS.datasets"."users.json" createDatasetFromSQL("SELECT * FROM \"testNAS.datasets\".\"users.json\"", asList("cp")); }