@Override public SharedResourceFactoryResponse<BasicDataSource> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, MysqlDataSourceKey> config) throws NotConfiguredException { MysqlDataSourceKey key = config.getKey(); Config configuration = key.getConfig(); BasicDataSource dataSource = MysqlStateStore.newDataSource(configuration); return new ResourceInstance<>(dataSource); }
@Override public SharedResourceFactoryResponse<FileSystem> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, FileSystemKey> config) throws NotConfiguredException { try { FileSystemKey key = config.getKey(); URI uri = key.getUri(); Configuration hadoopConf = key.getConfiguration(); log.info("Creating instrumented FileSystem for uri " + uri); Class<? extends FileSystem> fsClass = FileSystem.getFileSystemClass(uri.getScheme(), hadoopConf); if (InstrumentedFileSystem.class.isAssignableFrom(fsClass)) { InstrumentedFileSystem tmpfs = (InstrumentedFileSystem) fsClass.newInstance(); hadoopConf = new Configuration(hadoopConf); String schemeKey = "fs." + uri.getScheme() + ".impl"; hadoopConf.set(schemeKey, tmpfs.underlyingFs.getClass().getName()); } FileSystem fs = FileSystem.newInstance(uri, hadoopConf); ServiceLoader<FileSystemInstrumentationFactory> loader = ServiceLoader.load(FileSystemInstrumentationFactory.class); for (FileSystemInstrumentationFactory instrumentationFactory : loader) { fs = instrumentationFactory.instrumentFileSystem(fs, broker, config); } return new ResourceInstance<>(fs); } catch (IOException | ReflectiveOperationException ioe) { throw new RuntimeException(ioe); } }
@Override public SharedResourceFactoryResponse<HiveConf> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, SharedHiveConfKey> config) throws NotConfiguredException { SharedHiveConfKey sharedHiveConfKey = config.getKey(); HiveConf rawConf = new HiveConf(); if (!sharedHiveConfKey.hiveConfUri.equals(SharedHiveConfKey.INSTANCE.toConfigurationKey()) && StringUtils .isNotEmpty(sharedHiveConfKey.hiveConfUri)) { rawConf.setVar(HiveConf.ConfVars.METASTOREURIS, sharedHiveConfKey.hiveConfUri); rawConf.set(HIVE_METASTORE_TOKEN_SIGNATURE, sharedHiveConfKey.hiveConfUri); } return new ResourceInstance<>(rawConf); }
@Override public SharedResourceFactoryResponse<RequestSender> createResource( SharedResourcesBroker<S> broker, ScopedConfigView<S, SharedRestClientKey> config) throws NotConfiguredException { try { List<String> connectionPrefixes = SharedRestClientFactory.parseConnectionPrefixes(config.getConfig(), config.getKey()); return new ResourceInstance<>( new RedirectAwareRestClientRequestSender(broker, connectionPrefixes)); } catch (URISyntaxException use) { throw new RuntimeException(use); } }
@Override public SharedResourceFactoryResponse<DataPublisher> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, DataPublisherKey> config) throws NotConfiguredException { try { DataPublisherKey key = config.getKey(); String publisherClassName = key.getPublisherClassName(); State state = key.getState(); Class<? extends DataPublisher> dataPublisherClass = (Class<? extends DataPublisher>) Class .forName(publisherClassName); log.info("Creating data publisher with class {} in scope {}. ", publisherClassName, config.getScope().toString()); DataPublisher publisher = DataPublisher.getInstance(dataPublisherClass, state); // If the publisher is threadsafe then it is shareable, so return it as a resource instance that may be cached // by the broker. // Otherwise, it is not shareable, so return it as an immediately invalidated resource that will only be returned // once from the broker. if (isPublisherCacheable(publisher)) { return new ResourceInstance<>(publisher); } else { return new ImmediatelyInvalidResourceEntry<>(publisher); } } catch (ReflectiveOperationException e) { throw new RuntimeException(e); } }
@Override public SharedResourceFactoryResponse<RestClient> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, SharedRestClientKey> config) throws NotConfiguredException { try { SharedRestClientKey key = config.getKey(); if (!(key instanceof UriRestClientKey)) { return new ResourceCoordinate<>(this, new UriRestClientKey(key.serviceName, resolveUriPrefix(config.getConfig(), key)), config.getScope()); } String uriPrefix = ((UriRestClientKey) key).getUri(); HttpClientFactory http = new HttpClientFactory(FilterChains.empty(), new NioEventLoopGroup(0 /* use default settings */, ExecutorsUtils.newDaemonThreadFactory(Optional.<Logger>absent(), Optional.of("R2 Nio Event Loop-%d"))), true, Executors.newSingleThreadScheduledExecutor( ExecutorsUtils.newDaemonThreadFactory(Optional.<Logger>absent(), Optional.of("R2 Netty Scheduler"))), true); Client r2Client = new TransportClientAdapter(http.getClient(Collections.<String, String>emptyMap())); return new ResourceInstance<>(new RestClient(r2Client,uriPrefix)); } catch (URISyntaxException use) { throw new RuntimeException("Could not create a rest client for key " + Optional.fromNullable(config.getKey().toConfigurationKey()).or("null")); } }
@Override public SharedResourceFactoryResponse<ThrottlingPolicy> createResource(SharedResourcesBroker<ThrottlingServerScopes> broker, ScopedConfigView<ThrottlingServerScopes, SharedLimiterKey> configView) throws NotConfiguredException { Config config = configView.getConfig(); if (!config.hasPath(POLICY_KEY)) { if (config.hasPath(FAIL_ON_UNKNOWN_RESOURCE_ID) && config.getBoolean(FAIL_ON_UNKNOWN_RESOURCE_ID)) { throw new NotConfiguredException("Missing key " + POLICY_KEY); } else { return new TTLResourceEntry<ThrottlingPolicy>(new NoopPolicy(), ConfigUtils.getLong(config, RELOAD_FREQUENCY_KEY, DEFAULT_RELOAD_FREQUENCY), false); } } try { SpecificPolicyFactory factory = POLICY_CLASS_RESOLVER.resolveClass(config.getString(POLICY_KEY)).newInstance(); return new TTLResourceEntry<>(factory.createPolicy(configView.getKey(), broker, config), ConfigUtils.getLong(config, RELOAD_FREQUENCY_KEY, DEFAULT_RELOAD_FREQUENCY), false); } catch (ReflectiveOperationException roe) { throw new RuntimeException(roe); } }
SharedLimiterKey.GlobalLimiterPolicy globalLimiterPolicy = configView.getKey().getGlobalLimiterPolicy(); SharedLimiterKey modifiedKey = new SharedLimiterKey(configView.getKey().getResourceLimitedPath(), SharedLimiterKey.GlobalLimiterPolicy.LOCAL_ONLY); return new ResourceCoordinate<>(this, modifiedKey, (S) configView.getScope()); SharedLimiterKey modifiedKey = new SharedLimiterKey(configView.getKey().getResourceLimitedPath(), SharedLimiterKey.GlobalLimiterPolicy.USE_GLOBAL); return new ResourceCoordinate<>(this, modifiedKey, (S) configView.getScope()); Class<?> klazz = Class.forName("org.apache.gobblin.util.limiter.RestliLimiterFactory"); return new ResourceCoordinate<>((SharedResourceFactory<Limiter, SharedLimiterKey, S>) klazz.newInstance(), configView.getKey(), (S) configView.getScope()); } catch (ReflectiveOperationException roe) { if (globalLimiterPolicy.equals(SharedLimiterKey.GlobalLimiterPolicy.USE_GLOBAL)) { for (S parentScope : parentScopes) { limiter = new MultiLimiter(limiter, broker.getSharedResourceAtScope(this, configView.getKey(), parentScope));
if (config.getKey() instanceof SubTaggedMetricContextKey) { SubTaggedMetricContextKey key = (SubTaggedMetricContextKey) config.getKey(); if (parents != null && !parents.isEmpty()) { S parentScope = parents.iterator().next(); parentMetricContext = broker.getSharedResourceAtScope(this, config.getKey(), parentScope);
@Override public SharedResourceFactoryResponse<RestliServiceBasedLimiter> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, SharedLimiterKey> config) throws NotConfiguredException { S scope = config.getScope(); if (scope != scope.rootScope()) { return new ResourceCoordinate<>(this, config.getKey(), scope.rootScope()); } String serviceIdentifier = config.getConfig().hasPath(SERVICE_IDENTIFIER_KEY) ? config.getConfig().getString(SERVICE_IDENTIFIER_KEY) : "UNKNOWN"; String resourceLimited = config.getKey().getResourceLimitedPath(); MetricContextKey metricContextKey = new SubTaggedMetricContextKey(RestliServiceBasedLimiter.class.getSimpleName() + "_" + resourceLimited, ImmutableMap.of("resourceLimited", resourceLimited)); return new ResourceInstance<>( RestliServiceBasedLimiter.builder() .resourceLimited(resourceLimited) .serviceIdentifier(serviceIdentifier) .metricContext(broker.getSharedResource(new MetricContextFactory<S>(), metricContextKey)) .requestSender(broker.getSharedResource(new RedirectAwareRestClientRequestSender.Factory<S>(), new SharedRestClientKey(RESTLI_SERVICE_NAME))) .build() ); }
@Override public SharedResourceFactoryResponse<SharedResource> createResource(SharedResourcesBroker broker, ScopedConfigView config) { return new ResourceCoordinate<>(new TestFactory<S>(), (TestResourceKey) config.getKey(), (S) config.getScope().rootScope()); } }
@Override public SharedResourceFactoryResponse<SharedResource> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, TestResourceKey> config) { return new ResourceInstance<>(new SharedResource(config.getKey().getKey(), config.getConfig())); }
@Override public SharedResourceFactoryResponse<FileSystem> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, FileSystemKey> config) throws NotConfiguredException { try { FileSystemKey key = config.getKey(); URI uri = key.getUri(); Configuration hadoopConf = key.getConfiguration(); log.info("Creating instrumented FileSystem for uri " + uri); Class<? extends FileSystem> fsClass = FileSystem.getFileSystemClass(uri.getScheme(), hadoopConf); if (InstrumentedFileSystem.class.isAssignableFrom(fsClass)) { InstrumentedFileSystem tmpfs = (InstrumentedFileSystem) fsClass.newInstance(); hadoopConf = new Configuration(hadoopConf); String schemeKey = "fs." + uri.getScheme() + ".impl"; hadoopConf.set(schemeKey, tmpfs.underlyingFs.getClass().getName()); } FileSystem fs = FileSystem.newInstance(uri, hadoopConf); ServiceLoader<FileSystemInstrumentationFactory> loader = ServiceLoader.load(FileSystemInstrumentationFactory.class); for (FileSystemInstrumentationFactory instrumentationFactory : loader) { fs = instrumentationFactory.instrumentFileSystem(fs, broker, config); } return new ResourceInstance<>(fs); } catch (IOException | ReflectiveOperationException ioe) { throw new RuntimeException(ioe); } }
@Override public SharedResourceFactoryResponse<BasicDataSource> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, MysqlDataSourceKey> config) throws NotConfiguredException { MysqlDataSourceKey key = config.getKey(); Config configuration = key.getConfig(); BasicDataSource dataSource = MysqlStateStore.newDataSource(configuration); return new ResourceInstance<>(dataSource); }
@Override public SharedResourceFactoryResponse<HiveConf> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, SharedHiveConfKey> config) throws NotConfiguredException { SharedHiveConfKey sharedHiveConfKey = config.getKey(); HiveConf rawConf = new HiveConf(); if (!sharedHiveConfKey.hiveConfUri.equals(SharedHiveConfKey.INSTANCE.toConfigurationKey()) && StringUtils .isNotEmpty(sharedHiveConfKey.hiveConfUri)) { rawConf.setVar(HiveConf.ConfVars.METASTOREURIS, sharedHiveConfKey.hiveConfUri); rawConf.set(HIVE_METASTORE_TOKEN_SIGNATURE, sharedHiveConfKey.hiveConfUri); } return new ResourceInstance<>(rawConf); }
@Override public SharedResourceFactoryResponse<DataPublisher> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, DataPublisherKey> config) throws NotConfiguredException { try { DataPublisherKey key = config.getKey(); String publisherClassName = key.getPublisherClassName(); State state = key.getState(); Class<? extends DataPublisher> dataPublisherClass = (Class<? extends DataPublisher>) Class .forName(publisherClassName); log.info("Creating data publisher with class {} in scope {}. ", publisherClassName, config.getScope().toString()); DataPublisher publisher = DataPublisher.getInstance(dataPublisherClass, state); // If the publisher is threadsafe then it is shareable, so return it as a resource instance that may be cached // by the broker. // Otherwise, it is not shareable, so return it as an immediately invalidated resource that will only be returned // once from the broker. if (isPublisherCacheable(publisher)) { return new ResourceInstance<>(publisher); } else { return new ImmediatelyInvalidResourceEntry<>(publisher); } } catch (ReflectiveOperationException e) { throw new RuntimeException(e); } }
@Override public SharedResourceFactoryResponse<ThrottlingPolicy> createResource(SharedResourcesBroker<ThrottlingServerScopes> broker, ScopedConfigView<ThrottlingServerScopes, SharedLimiterKey> configView) throws NotConfiguredException { Config config = configView.getConfig(); if (!config.hasPath(POLICY_KEY)) { if (config.hasPath(FAIL_ON_UNKNOWN_RESOURCE_ID) && config.getBoolean(FAIL_ON_UNKNOWN_RESOURCE_ID)) { throw new NotConfiguredException("Missing key " + POLICY_KEY); } else { return new TTLResourceEntry<ThrottlingPolicy>(new NoopPolicy(), ConfigUtils.getLong(config, RELOAD_FREQUENCY_KEY, DEFAULT_RELOAD_FREQUENCY), false); } } try { SpecificPolicyFactory factory = POLICY_CLASS_RESOLVER.resolveClass(config.getString(POLICY_KEY)).newInstance(); return new TTLResourceEntry<>(factory.createPolicy(configView.getKey(), broker, config), ConfigUtils.getLong(config, RELOAD_FREQUENCY_KEY, DEFAULT_RELOAD_FREQUENCY), false); } catch (ReflectiveOperationException roe) { throw new RuntimeException(roe); } }
@Override public SharedResourceFactoryResponse<RestClient> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, SharedRestClientKey> config) throws NotConfiguredException { try { SharedRestClientKey key = config.getKey(); if (!(key instanceof UriRestClientKey)) { return new ResourceCoordinate<>(this, new UriRestClientKey(key.serviceName, resolveUriPrefix(config.getConfig(), key)), config.getScope()); } String uriPrefix = ((UriRestClientKey) key).getUri(); HttpClientFactory http = new HttpClientFactory(FilterChains.empty(), new NioEventLoopGroup(0 /* use default settings */, ExecutorsUtils.newDaemonThreadFactory(Optional.<Logger>absent(), Optional.of("R2 Nio Event Loop-%d"))), true, Executors.newSingleThreadScheduledExecutor( ExecutorsUtils.newDaemonThreadFactory(Optional.<Logger>absent(), Optional.of("R2 Netty Scheduler"))), true); Client r2Client = new TransportClientAdapter(http.getClient(Collections.<String, String>emptyMap())); return new ResourceInstance<>(new RestClient(r2Client,uriPrefix)); } catch (URISyntaxException use) { throw new RuntimeException("Could not create a rest client for key " + Optional.fromNullable(config.getKey().toConfigurationKey()).or("null")); } }
SharedLimiterKey.GlobalLimiterPolicy globalLimiterPolicy = configView.getKey().getGlobalLimiterPolicy(); SharedLimiterKey modifiedKey = new SharedLimiterKey(configView.getKey().getResourceLimitedPath(), SharedLimiterKey.GlobalLimiterPolicy.LOCAL_ONLY); return new ResourceCoordinate<>(this, modifiedKey, (S) configView.getScope()); SharedLimiterKey modifiedKey = new SharedLimiterKey(configView.getKey().getResourceLimitedPath(), SharedLimiterKey.GlobalLimiterPolicy.USE_GLOBAL); return new ResourceCoordinate<>(this, modifiedKey, (S) configView.getScope()); Class<?> klazz = Class.forName("org.apache.gobblin.util.limiter.RestliLimiterFactory"); return new ResourceCoordinate<>((SharedResourceFactory<Limiter, SharedLimiterKey, S>) klazz.newInstance(), configView.getKey(), (S) configView.getScope()); } catch (ReflectiveOperationException roe) { if (globalLimiterPolicy.equals(SharedLimiterKey.GlobalLimiterPolicy.USE_GLOBAL)) { for (S parentScope : parentScopes) { limiter = new MultiLimiter(limiter, broker.getSharedResourceAtScope(this, configView.getKey(), parentScope));
if (config.getKey() instanceof SubTaggedMetricContextKey) { SubTaggedMetricContextKey key = (SubTaggedMetricContextKey) config.getKey(); if (parents != null && !parents.isEmpty()) { S parentScope = parents.iterator().next(); parentMetricContext = broker.getSharedResourceAtScope(this, config.getKey(), parentScope);