public static SourceConfig create( String name, URI path, SchemaMutability mutability, MetadataPolicy policy ) { SourceConfig conf = new SourceConfig(); final String connection; if(path.getAuthority() != null) { connection = path.getScheme() + "://" + path.getAuthority() + "/"; } else { connection = path.getScheme() + ":///"; } InternalFileConf fc = new InternalFileConf(connection, path.getPath(), false, null, mutability); conf.setConnectionConf(fc); conf.setMetadataPolicy(policy); conf.setName(name); return conf; }
InternalFileConf.create(DACDaemonModule.JOBS_STORAGEPLUGIN_NAME, resultsPath, SchemaMutability.SYSTEM_TABLE, CatalogService.NEVER_REFRESH_POLICY_WITH_AUTO_PROMOTE)); InternalFileConf.create(DACDaemonModule.SCRATCH_STORAGEPLUGIN_NAME, scratchPath, SchemaMutability.USER_TABLE, CatalogService.NEVER_REFRESH_POLICY_WITH_AUTO_PROMOTE)); InternalFileConf.create(DATASET_DOWNLOAD_STORAGE_PLUGIN, downloadPath, SchemaMutability.USER_TABLE, CatalogService.NEVER_REFRESH_POLICY)); InternalFileConf.create(LOGS_STORAGE_PLUGIN, logsPath, SchemaMutability.NONE, CatalogService.DEFAULT_METADATA_POLICY)); InternalFileConf.create(LOCAL_STORAGE_PLUGIN, supportURI, SchemaMutability.SYSTEM_TABLE, CatalogService.NEVER_REFRESH_POLICY));
public static void updateDfsTestTmpSchemaLocation(final CatalogServiceImpl catalog, final String tmpDirPath) throws ExecutionSetupException { final ManagedStoragePlugin msp = catalog.getManagedSource(DFS_TEST_PLUGIN_NAME); final FileSystemPlugin plugin = (FileSystemPlugin) catalog.getSource(DFS_TEST_PLUGIN_NAME); SourceConfig newConfig = msp.getId().getClonedConfig(); InternalFileConf conf = (InternalFileConf) plugin.getConfig(); conf.path = tmpDirPath; conf.mutability = SchemaMutability.ALL; newConfig.setConfig(conf.toBytesString()); catalog.getSystemUserCatalog().updateSource(newConfig); } }
InternalFileConf conf = new InternalFileConf(); conf.connection = "classpath:///"; conf.path = "/"; InternalFileConf updatedCConf = new InternalFileConf(); updatedCConf.connection = "file:///"; updatedCConf.path = "/"; InternalFileConf decryptedUpdatedConfig = (InternalFileConf) reader.getConnectionConf(updatedConfig); assertNotNull(decryptedConf.getProperties()); assertEquals(2, decryptedConf.getProperties().size()); assertTrue(decryptedUpdatedConfig.getProperties().isEmpty()); assertNotEquals(config.getMetadataPolicy(), updatedConfig.getMetadataPolicy()); assertNotEquals(decryptedConf.getConnection(), decryptedUpdatedConfig.getConnection()); assertEquals("file:///", decryptedUpdatedConfig.getConnection()); assertNotEquals(decryptedConf.isInternal, decryptedUpdatedConfig.isInternal); assertEquals(decryptedConf.path, decryptedUpdatedConfig.path); InternalFileConf updatedCConf2 = new InternalFileConf(); updatedCConf2.connection = "file:///"; updatedCConf2.path = "/"; assertTrue(decryptedConf2.getProperties().isEmpty());
private static SourceConfig cp() { SourceConfig c = new SourceConfig(); InternalFileConf conf = new InternalFileConf(); conf.connection = "classpath:///"; conf.path = "/"; conf.isInternal = false; c.setName("cp"); c.setConnectionConf(conf); c.setMetadataPolicy(CatalogService.NEVER_REFRESH_POLICY_WITH_AUTO_PROMOTE); return c; }
private static void addSubPathDfsPlugin() throws Exception { final CatalogServiceImpl pluginRegistry = (CatalogServiceImpl) getSabotContext().getCatalogService(); final ManagedStoragePlugin msp = pluginRegistry.getManagedSource("dfs_test"); StoragePluginId pluginId = msp.getId(); InternalFileConf nasConf = pluginId.getConnectionConf(); nasConf.path = storageBase.getPath(); nasConf.mutability = SchemaMutability.ALL; SourceConfig config = pluginId.getConfig(); config.setId(null); config.setTag(null); config.setConfigOrdinal(null); config.setName("subPathDfs"); config.setConfig(nasConf.toBytesString()); pluginRegistry.getSystemUserCatalog().createSource(config); }
public void addCPSource(boolean createFolder) throws Exception { getSourceService().registerSourceWithRuntime(InternalFileConf.create(TEST_SOURCE, new URI("classpath:///acceleration/"), SchemaMutability.ALL, CatalogService.DEFAULT_METADATA_POLICY)); final NamespaceService nsService = getNamespaceService(); final SpaceConfig config = new SpaceConfig().setName(TEST_SPACE); if(!nsService.exists(new SpacePath(config.getName()).toNamespaceKey())) { nsService.addOrUpdateSpace(new SpacePath(config.getName()).toNamespaceKey(), config); } if (createFolder) { final FolderConfig folderConfig = new FolderConfig().setName(TEST_FOLDER); nsService.addOrUpdateFolder(TEST_FOLDER_PATH.toNamespaceKey(), folderConfig); } }
protected static void addMiniDfsBasedStorage(final Map<String, WorkspaceConfig> workspaces, boolean impersonationEnabled) throws Exception { // Create a HDFS based storage plugin (connection string for mini dfs is varies for each run). final CatalogService catalogService = getSabotContext().getCatalogService(); final Path dirPath = new Path("/"); FileSystem.mkdirs(fs, dirPath, new FsPermission((short)0777)); fs.setOwner(dirPath, processUser, processUser); final InternalFileConf conf = new InternalFileConf(); conf.connection = fs.getUri().toString(); conf.path = "/"; conf.enableImpersonation = impersonationEnabled; conf.mutability = SchemaMutability.ALL; final SourceConfig config = new SourceConfig(); config.setName(MINIDFS_STORAGE_PLUGIN_NAME); config.setConnectionConf(conf); config.setMetadataPolicy(CatalogService.DEFAULT_METADATA_POLICY_WITH_AUTO_PROMOTE); ((CatalogServiceImpl) catalogService).getSystemUserCatalog().createSource(config); }
InternalFileConf conf = new InternalFileConf(); conf.connection = "file:///"; conf.path = "/"; InternalFileConf conf = new InternalFileConf(); conf.connection = "file:///"; conf.path = tmpDirPath; InternalFileConf conf = new InternalFileConf(); conf.connection = "file:///"; conf.path = "/"; InternalFileConf conf = new InternalFileConf(); conf.connection = "file:///"; conf.path = "/";
) { SourceConfig conf = new SourceConfig(); InternalFileConf fc = new InternalFileConf(connection, path, enableImpersonation, properties, mutability); conf.setConnectionConf(fc); conf.setMetadataPolicy(policy);
InternalFileConf conf = new InternalFileConf(); conf.connection = "file:///"; conf.path = getDfsTestTmpSchemaLocation();