@Override public HomeFileConf getConfig() { return super.getConfig(); }
ReflectionManager(SabotContext sabotContext, JobsService jobsService, NamespaceService namespaceService, OptionManager optionManager, ReflectionGoalsStore userStore, ReflectionEntriesStore reflectionStore, ExternalReflectionStore externalReflectionStore, MaterializationStore materializationStore, DependencyManager dependencyManager, DescriptorCache descriptorCache, Set<ReflectionId> reflectionsToUpdate, WakeUpCallback wakeUpCallback, Supplier<ExpansionHelper> expansionHelper) { this.sabotContext = Preconditions.checkNotNull(sabotContext, "sabotContext required"); this.jobsService = Preconditions.checkNotNull(jobsService, "jobsService required"); this.namespaceService = Preconditions.checkNotNull(namespaceService, "namespaceService required"); this.optionManager = Preconditions.checkNotNull(optionManager, "optionManager required"); this.userStore = Preconditions.checkNotNull(userStore, "reflection user store required"); this.reflectionStore = Preconditions.checkNotNull(reflectionStore, "reflection store required"); this.externalReflectionStore = Preconditions.checkNotNull(externalReflectionStore); this.materializationStore = Preconditions.checkNotNull(materializationStore, "materialization store required"); this.dependencyManager = Preconditions.checkNotNull(dependencyManager, "dependency manager required"); this.descriptorCache = Preconditions.checkNotNull(descriptorCache, "descriptor cache required"); this.reflectionsToUpdate = Preconditions.checkNotNull(reflectionsToUpdate, "reflections to update required"); this.wakeUpCallback = Preconditions.checkNotNull(wakeUpCallback, "wakeup callback required"); this.expansionHelper = Preconditions.checkNotNull(expansionHelper, "sqlConvertSupplier required"); final FileSystemPlugin accelerationPlugin = sabotContext.getCatalogService() .getSource(ReflectionServiceImpl.ACCELERATOR_STORAGEPLUGIN_NAME); accelerationBasePath = accelerationPlugin.getConfig().getPath(); }
public JobResultsStore(final FileSystemPlugin plugin, final IndexedStore<JobId, JobResult> store, final BufferAllocator allocator) throws IOException { this.storageName = plugin.getName(); this.dfs = plugin.getFS(ImpersonationUtil.getProcessUserName()); this.jobStoreLocation = plugin.getConfig().getPath(); this.dfs.mkdirs(jobStoreLocation); this.store = store; this.allocator = allocator; this.jobResults = CacheBuilder.newBuilder() .maximumSize(100) .expireAfterAccess(15, TimeUnit.MINUTES) .build( new CacheLoader<JobId, JobData>() { @Override public JobData load(JobId key) throws Exception { final JobDataImpl jobDataImpl = new JobDataImpl(new LateJobLoader(key), key); return newJobDataReference(jobDataImpl); } }); }
@Override public void start() throws Exception { store = new ConfigurationStore(kvStoreProvider.get()); ClusterIdentity identity; Optional<ClusterIdentity> clusterIdentity = getClusterIdentityFromStore(store, kvStoreProvider.get()); if (!clusterIdentity.isPresent()) { // this is a new cluster, generating a new cluster identifier. identity = new ClusterIdentity() .setIdentity(UUID.randomUUID().toString()) .setVersion(toClusterVersion(VERSION)) .setCreated(System.currentTimeMillis()); identity = storeIdentity(identity); } else { identity = clusterIdentity.get(); } this.identity = identity; FileSystemPlugin supportPlugin = catalogServiceProvider.get().getSource(LOCAL_STORAGE_PLUGIN); Preconditions.checkNotNull(supportPlugin); final String supportPathURI = supportPlugin.getConfig().getPath().toString(); supportPath = new File(supportPathURI).toPath(); }
private DatasetDownloadManager downloadManager() { final FileSystemPlugin downloadPlugin = catalogService.getSource(DATASET_DOWNLOAD_STORAGE_PLUGIN); return new DatasetDownloadManager(jobsService, namespaceService, downloadPlugin.getConfig().getPath(), downloadPlugin.getFs()); } private void validate(DatasetPath path, VirtualDatasetUI ds) {
public static void updateDfsTestTmpSchemaLocation(final CatalogServiceImpl catalog, final String tmpDirPath) throws ExecutionSetupException { final ManagedStoragePlugin msp = catalog.getManagedSource(DFS_TEST_PLUGIN_NAME); final FileSystemPlugin plugin = (FileSystemPlugin) catalog.getSource(DFS_TEST_PLUGIN_NAME); SourceConfig newConfig = msp.getId().getClonedConfig(); InternalFileConf conf = (InternalFileConf) plugin.getConfig(); conf.path = tmpDirPath; conf.mutability = SchemaMutability.ALL; newConfig.setConfig(conf.toBytesString()); catalog.getSystemUserCatalog().updateSource(newConfig); } }
@Test public void testPDFSFileStore() throws Exception { FileSystemPlugin fsp = (FileSystemPlugin) l(CatalogService.class).getSource(HomeFileSystemStoragePlugin.HOME_PLUGIN_NAME); HomeFileConf conf = (HomeFileConf) fsp.getConfig(); HomeFileTool tool = l(HomeFileTool.class); try { runTests(conf); } finally { tool.clear(); } } }
@Test public void testCTASAndDropTable() throws Exception { // Create a table SqlQuery ctas = getQueryFromSQL("CREATE TABLE \"$scratch\".\"ctas\" AS select * from cp.\"json/users.json\" LIMIT 1"); Job ctasJob = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(ctas) .setQueryType(QueryType.UI_RUN) .build(), NoOpJobStatusListener.INSTANCE); ctasJob.getData().loadIfNecessary(); FileSystemPlugin plugin = (FileSystemPlugin) getCurrentDremioDaemon().getBindingProvider().lookup(CatalogService.class).getSource("$scratch"); // Make sure the table data files exist File ctasTableDir = new File(plugin.getConfig().getPath().toString(), "ctas"); assertTrue(ctasTableDir.exists()); assertTrue(ctasTableDir.list().length >= 1); // Now drop the table SqlQuery dropTable = getQueryFromSQL("DROP TABLE \"$scratch\".\"ctas\""); Job dropTableJob = jobsService.submitJob(JobRequest.newBuilder() .setSqlQuery(dropTable) .setQueryType(QueryType.ACCELERATOR_DROP) .build(), NoOpJobStatusListener.INSTANCE); dropTableJob.getData().loadIfNecessary(); // Make sure the table data directory is deleted assertFalse(ctasTableDir.exists()); }
final File ctasTableDir = new File(plugin.getConfig().getPath().toString(), "ctas"); assertTrue(ctasTableDir.exists()); assertTrue(ctasTableDir.list().length >= 1);