/** * Equivalent to {@link FileSystem#get(URI, Configuration)}, but uses the input {@link SharedResourcesBroker} to configure * add-ons to the {@link FileSystem} (e.g. throttling, instrumentation). */ public static <S extends ScopeType<S>> FileSystem get(URI uri, Configuration configuration, SharedResourcesBroker<S> broker) throws IOException { try { return broker.getSharedResource(new FileSystemFactory<S>(), new FileSystemKey(uri, configuration)); } catch (NotConfiguredException nce) { throw new IOException(nce); } }
@Override public SharedResourceFactoryResponse<FileSystem> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, FileSystemKey> config) throws NotConfiguredException { try { FileSystemKey key = config.getKey(); URI uri = key.getUri(); Configuration hadoopConf = key.getConfiguration(); log.info("Creating instrumented FileSystem for uri " + uri); Class<? extends FileSystem> fsClass = FileSystem.getFileSystemClass(uri.getScheme(), hadoopConf); if (InstrumentedFileSystem.class.isAssignableFrom(fsClass)) { InstrumentedFileSystem tmpfs = (InstrumentedFileSystem) fsClass.newInstance(); hadoopConf = new Configuration(hadoopConf); String schemeKey = "fs." + uri.getScheme() + ".impl"; hadoopConf.set(schemeKey, tmpfs.underlyingFs.getClass().getName()); } FileSystem fs = FileSystem.newInstance(uri, hadoopConf); ServiceLoader<FileSystemInstrumentationFactory> loader = ServiceLoader.load(FileSystemInstrumentationFactory.class); for (FileSystemInstrumentationFactory instrumentationFactory : loader) { fs = instrumentationFactory.instrumentFileSystem(fs, broker, config); } return new ResourceInstance<>(fs); } catch (IOException | ReflectiveOperationException ioe) { throw new RuntimeException(ioe); } }
public FileSystemKey(URI uri, Configuration configuration) { this.configuration = configuration; this.uri = resolveURI(uri, configuration); }
@Override public FileSystem instrumentFileSystem(FileSystem fs, SharedResourcesBroker<S> broker, ConfigView<S, FileSystemKey> config) { try { String serviceName = ConfigUtils.getString(config.getConfig(), SERVICE_NAME_CONF_KEY, ""); Limiter limiter = broker.getSharedResource(new SharedLimiterFactory<S>(), new FileSystemLimiterKey(config.getKey().getUri())); return new ThrottledFileSystem(fs, limiter, serviceName); } catch (NotConfiguredException nce) { throw new RuntimeException(nce); } } }
@Override public FileSystem instrumentFileSystem(FileSystem fs, SharedResourcesBroker<S> broker, ConfigView<S, FileSystemKey> config) { try { String serviceName = ConfigUtils.getString(config.getConfig(), SERVICE_NAME_CONF_KEY, ""); Limiter limiter = broker.getSharedResource(new SharedLimiterFactory<S>(), new FileSystemLimiterKey(config.getKey().getUri())); return new ThrottledFileSystem(fs, limiter, serviceName); } catch (NotConfiguredException nce) { throw new RuntimeException(nce); } } }
@Test public void testCreationWithConfigurationFSImpl() throws Exception { SharedResourcesBrokerImpl<SimpleScopeType> broker = SharedResourcesBrokerFactory.<SimpleScopeType>createDefaultTopLevelBroker( ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance()); Configuration conf = new Configuration(); conf.set("fs.local.impl", InstrumentedLocalFileSystem.class.getName()); FileSystemKey key = new FileSystemKey(new URI("file:///"), new Configuration()); FileSystemFactory<SimpleScopeType> factory = new FileSystemFactory<>(); FileSystem fs = broker.getSharedResource(factory, key); verifyInstrumentedOnce(fs); Assert.assertTrue(DecoratorUtils.resolveUnderlyingObject(fs) instanceof LocalFileSystem); }
@Override public SharedResourceFactoryResponse<FileSystem> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, FileSystemKey> config) throws NotConfiguredException { try { FileSystemKey key = config.getKey(); URI uri = key.getUri(); Configuration hadoopConf = key.getConfiguration(); log.info("Creating instrumented FileSystem for uri " + uri); Class<? extends FileSystem> fsClass = FileSystem.getFileSystemClass(uri.getScheme(), hadoopConf); if (InstrumentedFileSystem.class.isAssignableFrom(fsClass)) { InstrumentedFileSystem tmpfs = (InstrumentedFileSystem) fsClass.newInstance(); hadoopConf = new Configuration(hadoopConf); String schemeKey = "fs." + uri.getScheme() + ".impl"; hadoopConf.set(schemeKey, tmpfs.underlyingFs.getClass().getName()); } FileSystem fs = FileSystem.newInstance(uri, hadoopConf); ServiceLoader<FileSystemInstrumentationFactory> loader = ServiceLoader.load(FileSystemInstrumentationFactory.class); for (FileSystemInstrumentationFactory instrumentationFactory : loader) { fs = instrumentationFactory.instrumentFileSystem(fs, broker, config); } return new ResourceInstance<>(fs); } catch (IOException | ReflectiveOperationException ioe) { throw new RuntimeException(ioe); } }
public FileSystemKey(URI uri, Configuration configuration) { this.configuration = configuration; this.uri = resolveURI(uri, configuration); }
@Test public void testCreationWithInstrumentedScheme() throws Exception { SharedResourcesBrokerImpl<SimpleScopeType> broker = SharedResourcesBrokerFactory.<SimpleScopeType>createDefaultTopLevelBroker( ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance()); FileSystemKey key = new FileSystemKey(new URI("instrumented-file:///"), new Configuration()); FileSystemFactory<SimpleScopeType> factory = new FileSystemFactory<>(); FileSystem fs = broker.getSharedResource(factory, key); verifyInstrumentedOnce(fs); Assert.assertTrue(DecoratorUtils.resolveUnderlyingObject(fs) instanceof LocalFileSystem); }
@Test public void test() throws Exception { SharedResourcesBrokerImpl<SimpleScopeType> broker = SharedResourcesBrokerFactory.<SimpleScopeType>createDefaultTopLevelBroker( ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance()); FileSystemKey key = new FileSystemKey(new URI("file:///"), new Configuration()); FileSystemFactory<SimpleScopeType> factory = new FileSystemFactory<>(); FileSystem fs = broker.getSharedResource(factory, key); verifyInstrumentedOnce(fs); SharedResourcesBroker<SimpleScopeType> subBroker = broker.newSubscopedBuilder(SimpleScopeType.LOCAL.defaultScopeInstance()).build(); FileSystem subBrokerFs = FileSystemFactory.get(new URI("file:///"), new Configuration(), subBroker); Assert.assertEquals(fs, subBrokerFs); }
/** * Equivalent to {@link FileSystem#get(URI, Configuration)}, but uses the input {@link SharedResourcesBroker} to configure * add-ons to the {@link FileSystem} (e.g. throttling, instrumentation). */ public static <S extends ScopeType<S>> FileSystem get(URI uri, Configuration configuration, SharedResourcesBroker<S> broker) throws IOException { try { return broker.getSharedResource(new FileSystemFactory<S>(), new FileSystemKey(uri, configuration)); } catch (NotConfiguredException nce) { throw new IOException(nce); } }