<T, K extends SharedResourceKey> void put(final SharedResourceFactory<T, K, S> factory, @Nonnull final K key, @Nonnull final ScopeWrapper<S> scope, T instance) { RawJobBrokerKey fullKey = new RawJobBrokerKey(scope, factory.getName(), key); this.sharedResourceCache.put(fullKey, new ResourceInstance<>(instance)); }
@Override public SharedResourceFactoryResponse<ForkThrowableHolder> createResource( SharedResourcesBroker<GobblinScopeTypes> broker, ScopedConfigView<GobblinScopeTypes, EmptyKey> config) throws NotConfiguredException { return new ResourceInstance<>(new ForkThrowableHolder()); }
@Override public SharedResourceFactoryResponse<StreamThrottler<S>> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, EmptyKey> config) throws NotConfiguredException { return new ResourceInstance<>(new StreamThrottler<>(broker)); }
@Override public SharedResourceFactoryResponse<Pusher<T>> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, StringNameSharedResourceKey> config) throws NotConfiguredException { Config pusherConfig = config.getConfig().withFallback(FALLBACK); String pusherClass = pusherConfig.getString(PUSHER_CLASS); Pusher<T> pusher; try { pusher = (Pusher) ConstructorUtils.invokeConstructor(Class.forName(pusherClass), pusherConfig); } catch (ReflectiveOperationException e) { log.warn("Unable to construct a pusher with class {}. LoggingPusher will be used", pusherClass, e); pusher = new LoggingPusher<>(); } return new ResourceInstance<>(pusher); } }
@Override public SharedResourceFactoryResponse<FileSystem> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, FileSystemKey> config) throws NotConfiguredException { try { FileSystemKey key = config.getKey(); URI uri = key.getUri(); Configuration hadoopConf = key.getConfiguration(); log.info("Creating instrumented FileSystem for uri " + uri); Class<? extends FileSystem> fsClass = FileSystem.getFileSystemClass(uri.getScheme(), hadoopConf); if (InstrumentedFileSystem.class.isAssignableFrom(fsClass)) { InstrumentedFileSystem tmpfs = (InstrumentedFileSystem) fsClass.newInstance(); hadoopConf = new Configuration(hadoopConf); String schemeKey = "fs." + uri.getScheme() + ".impl"; hadoopConf.set(schemeKey, tmpfs.underlyingFs.getClass().getName()); } FileSystem fs = FileSystem.newInstance(uri, hadoopConf); ServiceLoader<FileSystemInstrumentationFactory> loader = ServiceLoader.load(FileSystemInstrumentationFactory.class); for (FileSystemInstrumentationFactory instrumentationFactory : loader) { fs = instrumentationFactory.instrumentFileSystem(fs, broker, config); } return new ResourceInstance<>(fs); } catch (IOException | ReflectiveOperationException ioe) { throw new RuntimeException(ioe); } }
@Override public SharedResourceFactoryResponse<BasicDataSource> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, MysqlDataSourceKey> config) throws NotConfiguredException { MysqlDataSourceKey key = config.getKey(); Config configuration = key.getConfig(); BasicDataSource dataSource = MysqlStateStore.newDataSource(configuration); return new ResourceInstance<>(dataSource); }
@Override public SharedResourceFactoryResponse<LineageInfo> createResource(SharedResourcesBroker<GobblinScopeTypes> broker, ScopedConfigView<GobblinScopeTypes, EmptyKey> config) throws NotConfiguredException { return new ResourceInstance<>(new LineageInfo(config.getConfig())); }
@Override public SharedResourceFactoryResponse<HiveConf> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, SharedHiveConfKey> config) throws NotConfiguredException { SharedHiveConfKey sharedHiveConfKey = config.getKey(); HiveConf rawConf = new HiveConf(); if (!sharedHiveConfKey.hiveConfUri.equals(SharedHiveConfKey.INSTANCE.toConfigurationKey()) && StringUtils .isNotEmpty(sharedHiveConfKey.hiveConfUri)) { rawConf.setVar(HiveConf.ConfVars.METASTOREURIS, sharedHiveConfKey.hiveConfUri); rawConf.set(HIVE_METASTORE_TOKEN_SIGNATURE, sharedHiveConfKey.hiveConfUri); } return new ResourceInstance<>(rawConf); }
@Override public SharedResourceFactoryResponse<RequestSender> createResource( SharedResourcesBroker<S> broker, ScopedConfigView<S, SharedRestClientKey> config) throws NotConfiguredException { try { List<String> connectionPrefixes = SharedRestClientFactory.parseConnectionPrefixes(config.getConfig(), config.getKey()); return new ResourceInstance<>( new RedirectAwareRestClientRequestSender(broker, connectionPrefixes)); } catch (URISyntaxException use) { throw new RuntimeException(use); } }
@Override public SharedResourceFactoryResponse<DataPublisher> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, DataPublisherKey> config) throws NotConfiguredException { try { DataPublisherKey key = config.getKey(); String publisherClassName = key.getPublisherClassName(); State state = key.getState(); Class<? extends DataPublisher> dataPublisherClass = (Class<? extends DataPublisher>) Class .forName(publisherClassName); log.info("Creating data publisher with class {} in scope {}. ", publisherClassName, config.getScope().toString()); DataPublisher publisher = DataPublisher.getInstance(dataPublisherClass, state); // If the publisher is threadsafe then it is shareable, so return it as a resource instance that may be cached // by the broker. // Otherwise, it is not shareable, so return it as an immediately invalidated resource that will only be returned // once from the broker. if (isPublisherCacheable(publisher)) { return new ResourceInstance<>(publisher); } else { return new ImmediatelyInvalidResourceEntry<>(publisher); } } catch (ReflectiveOperationException e) { throw new RuntimeException(e); } }
@Override public ResourceInstance<MySharedObject> createResource(SharedResourcesBroker<GobblinScopeTypes> broker, ScopedConfigView<GobblinScopeTypes, MySharedKey> config) { return new ResourceInstance<>(new MySharedObject()); }
@Override public SharedResourceFactoryResponse<RestClient> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, SharedRestClientKey> config) throws NotConfiguredException { try { SharedRestClientKey key = config.getKey(); if (!(key instanceof UriRestClientKey)) { return new ResourceCoordinate<>(this, new UriRestClientKey(key.serviceName, resolveUriPrefix(config.getConfig(), key)), config.getScope()); } String uriPrefix = ((UriRestClientKey) key).getUri(); HttpClientFactory http = new HttpClientFactory(FilterChains.empty(), new NioEventLoopGroup(0 /* use default settings */, ExecutorsUtils.newDaemonThreadFactory(Optional.<Logger>absent(), Optional.of("R2 Nio Event Loop-%d"))), true, Executors.newSingleThreadScheduledExecutor( ExecutorsUtils.newDaemonThreadFactory(Optional.<Logger>absent(), Optional.of("R2 Netty Scheduler"))), true); Client r2Client = new TransportClientAdapter(http.getClient(Collections.<String, String>emptyMap())); return new ResourceInstance<>(new RestClient(r2Client,uriPrefix)); } catch (URISyntaxException use) { throw new RuntimeException("Could not create a rest client for key " + Optional.fromNullable(config.getKey().toConfigurationKey()).or("null")); } }
builder.addTag(new Tag<>(entry.getKey(), entry.getValue())); return new ResourceInstance<>(builder.build()); return new ResourceInstance<>(builder.build()); } catch (NoSuchScopeException nsse) { throw new RuntimeException("Could not create MetricContext.", nsse);
return new ResourceInstance<>(limiter);
@Override public SharedResourceFactoryResponse<SharedResource> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, TestResourceKey> config) { return new ResourceInstance<>(new SharedResource(config.getKey().getKey(), config.getConfig())); }
@Override public SharedResourceFactoryResponse<RestliServiceBasedLimiter> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, SharedLimiterKey> config) throws NotConfiguredException { S scope = config.getScope(); if (scope != scope.rootScope()) { return new ResourceCoordinate<>(this, config.getKey(), scope.rootScope()); } String serviceIdentifier = config.getConfig().hasPath(SERVICE_IDENTIFIER_KEY) ? config.getConfig().getString(SERVICE_IDENTIFIER_KEY) : "UNKNOWN"; String resourceLimited = config.getKey().getResourceLimitedPath(); MetricContextKey metricContextKey = new SubTaggedMetricContextKey(RestliServiceBasedLimiter.class.getSimpleName() + "_" + resourceLimited, ImmutableMap.of("resourceLimited", resourceLimited)); return new ResourceInstance<>( RestliServiceBasedLimiter.builder() .resourceLimited(resourceLimited) .serviceIdentifier(serviceIdentifier) .metricContext(broker.getSharedResource(new MetricContextFactory<S>(), metricContextKey)) .requestSender(broker.getSharedResource(new RedirectAwareRestClientRequestSender.Factory<S>(), new SharedRestClientKey(RESTLI_SERVICE_NAME))) .build() ); }
<T, K extends SharedResourceKey> void put(final SharedResourceFactory<T, K, S> factory, @Nonnull final K key, @Nonnull final ScopeWrapper<S> scope, T instance) { RawJobBrokerKey fullKey = new RawJobBrokerKey(scope, factory.getName(), key); this.sharedResourceCache.put(fullKey, new ResourceInstance<>(instance)); }
@Override public SharedResourceFactoryResponse<ForkThrowableHolder> createResource( SharedResourcesBroker<GobblinScopeTypes> broker, ScopedConfigView<GobblinScopeTypes, EmptyKey> config) throws NotConfiguredException { return new ResourceInstance<>(new ForkThrowableHolder()); }
@Override public SharedResourceFactoryResponse<BasicDataSource> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, MysqlDataSourceKey> config) throws NotConfiguredException { MysqlDataSourceKey key = config.getKey(); Config configuration = key.getConfig(); BasicDataSource dataSource = MysqlStateStore.newDataSource(configuration); return new ResourceInstance<>(dataSource); }
@Override public SharedResourceFactoryResponse<LineageInfo> createResource(SharedResourcesBroker<GobblinScopeTypes> broker, ScopedConfigView<GobblinScopeTypes, EmptyKey> config) throws NotConfiguredException { return new ResourceInstance<>(new LineageInfo(config.getConfig())); }