@Override public FileSystem run() throws Exception { return dfsc.get(uri, conf, uniqueConnProps); } });
@Override public synchronized void run() { try { closeAll(true); } catch (IOException e) { logger.info("DremioFileSystemCache.closeAll() threw an exception\n", e); } } }
/** * This test is to make sure legacy scenarios through {@link DremioFileSystemCache} work as expected. */ @Test public void withoutUniqueConnProps() throws Exception { final DremioFileSystemCache dfsc = new DremioFileSystemCache(); final URI uri = URI.create("file:///path"); Configuration conf1 = new Configuration(); // get a filesystem FileSystem fs1 = dfsc.get(uri, conf1, null); // get the same with slight configuration change Configuration conf2 = new Configuration(conf1); conf2.set("blah", "boo"); FileSystem fs2 = dfsc.get(uri, conf2, null); // Make sure both are of same instance as we expect the Hadoop FileSystem to cache them assertTrue(fs1 == fs2); // now create a third filesystem as different user and make sure we got a different filesystem instance FileSystem fs3 = getAs("newUser", dfsc, uri, conf1, null); assertTrue(fs1 != fs3); }
/** * This test is to make sure legacy scenarios through {@link DremioFileSystemCache} work as expected. */ @Test public void withoutUniqueConnPropsWithCacheExplicitlyDisabled() throws Exception { final DremioFileSystemCache dfsc = new DremioFileSystemCache(); final URI uri = URI.create("file:///path"); Configuration conf1 = new Configuration(); final String disableCacheName = String.format("fs.%s.impl.disable.cache", uri.getScheme()); conf1.setBoolean(disableCacheName, true); // get a filesystem FileSystem fs1 = dfsc.get(uri, conf1, null); // get the same with slight configuration change Configuration conf2 = new Configuration(conf1); conf2.set("blah", "boo"); FileSystem fs2 = dfsc.get(uri, conf2, null); // Make sure different instance as we expect the Hadoop FileSystem to not cache them assertTrue(fs1 != fs2); // now create a third filesystem as different user and make sure we got a different filesystem instance FileSystem fs3 = getAs("newUser", dfsc, uri, conf1, null); assertTrue(fs1 != fs3); // make sure we got a different filesystem instance assertTrue(fs1 != fs3); }
@Test public void withUniqueConnProps() throws Exception { final DremioFileSystemCache dfsc = new DremioFileSystemCache(); final URI uri = URI.create("file:///path"); final List<String> uniqueProps = ImmutableList.of("prop1", "prop2"); Configuration conf1 = new Configuration(); FileSystem fs1 = dfsc.get(uri, conf1, uniqueProps); // get a fs with change in unique props Configuration conf2 = new Configuration(conf1); conf2.set("prop1", "prop1Val"); FileSystem fs2 = dfsc.get(uri, conf2, uniqueProps); // Expect a different filesystem assertTrue(fs1 != fs2); // Now get one more with same conf as fs2, expect to get the same entry as fs2 as it is cached and matches FileSystem fs3 = dfsc.get(uri, conf2, uniqueProps); assertTrue(fs2 == fs3); // now create a third filesystem as different user and make sure we got a different filesystem instance FileSystem fs4 = getAs("newUser", dfsc, uri, conf2, uniqueProps); assertTrue(fs2 != fs4); assertTrue(fs1 != fs4); // Now try to get a file system without unique props but same conf as fs1 FileSystem fs5 = dfsc.get(uri, conf1, null); assertTrue(fs1 != fs5); // as this is created by Hadoop FileSystem cache // Now get one more same as fs5 conf and expect it to be same as fs5 as it is retrieved from Hadoop FileSystem cache FileSystem fs6 = dfsc.get(uri, conf1, null); assertTrue(fs5 == fs6); }
public FileSystemWrapper(Configuration fsConf, OperatorStats operatorStats, List<String> connectionUniqueProps) throws IOException { this(fsConf, DREMIO_FS_CACHE.get(FileSystem.getDefaultUri(fsConf), fsConf, connectionUniqueProps), operatorStats); }