<T, K extends SharedResourceKey> void put(final SharedResourceFactory<T, K, S> factory, @Nonnull final K key, @Nonnull final ScopeWrapper<S> scope, T instance) { RawJobBrokerKey fullKey = new RawJobBrokerKey(scope, factory.getName(), key); this.sharedResourceCache.put(fullKey, new ResourceInstance<>(instance)); }
@Override public synchronized T getResource() { // mark the object as invalid before returning so that a new one will be created on the next // request from the factory this.valid = false; return super.getResource(); }
/** * This method returns the resource, but may have logic before the return. * @return the resource */ @Override public T getResource() { return getTheResource(); }
@Override public SharedResourceFactoryResponse<ForkThrowableHolder> createResource( SharedResourcesBroker<GobblinScopeTypes> broker, ScopedConfigView<GobblinScopeTypes, EmptyKey> config) throws NotConfiguredException { return new ResourceInstance<>(new ForkThrowableHolder()); }
Object obj = ((ResourceInstance) entry.getValue()).getResource();
/** * This method returns the resource, but may have logic before the return. * @return the resource */ @Override public T getResource() { return getTheResource(); }
@Override public SharedResourceFactoryResponse<Pusher<T>> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, StringNameSharedResourceKey> config) throws NotConfiguredException { Config pusherConfig = config.getConfig().withFallback(FALLBACK); String pusherClass = pusherConfig.getString(PUSHER_CLASS); Pusher<T> pusher; try { pusher = (Pusher) ConstructorUtils.invokeConstructor(Class.forName(pusherClass), pusherConfig); } catch (ReflectiveOperationException e) { log.warn("Unable to construct a pusher with class {}. LoggingPusher will be used", pusherClass, e); pusher = new LoggingPusher<>(); } return new ResourceInstance<>(pusher); } }
@Test public void testCountLimiter() throws Exception { SharedResourcesBrokerImpl<SimpleScopeType> broker = getBrokerForConfigMap(ImmutableMap.of( JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SharedLimiterFactory.LIMITER_CLASS_KEY), "CountBasedLimiter", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, CountBasedLimiter.Factory.COUNT_KEY), "10" )); SharedLimiterFactory<SimpleScopeType> factory = new SharedLimiterFactory<>(); Assert.assertEquals( factory.getAutoScope(broker, broker.getConfigView(SimpleScopeType.LOCAL, new SharedLimiterKey("resource"), factory.getName())), SimpleScopeType.GLOBAL); Limiter limiter = ((ResourceInstance<Limiter>) factory.createResource(broker, broker.getConfigView(SimpleScopeType.GLOBAL, new SharedLimiterKey("resource"), factory.getName()))).getResource(); Assert.assertTrue(limiter instanceof CountBasedLimiter); Assert.assertEquals(((CountBasedLimiter) limiter).getCountLimit(), 10); }
@Override public SharedResourceFactoryResponse<StreamThrottler<S>> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, EmptyKey> config) throws NotConfiguredException { return new ResourceInstance<>(new StreamThrottler<>(broker)); }
@Test public void testEmptyConfig() throws Exception { SharedResourcesBrokerImpl<SimpleScopeType> broker = getBrokerForConfigMap(ImmutableMap.<String, String>of()); SharedLimiterFactory<SimpleScopeType> factory = new SharedLimiterFactory<>(); Assert.assertEquals( factory.getAutoScope(broker, broker.getConfigView(SimpleScopeType.LOCAL, new SharedLimiterKey("resource"), factory.getName())), SimpleScopeType.GLOBAL); Limiter limiter = ((ResourceInstance<Limiter>) factory.createResource(broker, broker.getConfigView(SimpleScopeType.GLOBAL, new SharedLimiterKey("resource"), factory.getName()))).getResource(); Assert.assertTrue(limiter instanceof NoopLimiter); }
@Override public SharedResourceFactoryResponse<FileSystem> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, FileSystemKey> config) throws NotConfiguredException { try { FileSystemKey key = config.getKey(); URI uri = key.getUri(); Configuration hadoopConf = key.getConfiguration(); log.info("Creating instrumented FileSystem for uri " + uri); Class<? extends FileSystem> fsClass = FileSystem.getFileSystemClass(uri.getScheme(), hadoopConf); if (InstrumentedFileSystem.class.isAssignableFrom(fsClass)) { InstrumentedFileSystem tmpfs = (InstrumentedFileSystem) fsClass.newInstance(); hadoopConf = new Configuration(hadoopConf); String schemeKey = "fs." + uri.getScheme() + ".impl"; hadoopConf.set(schemeKey, tmpfs.underlyingFs.getClass().getName()); } FileSystem fs = FileSystem.newInstance(uri, hadoopConf); ServiceLoader<FileSystemInstrumentationFactory> loader = ServiceLoader.load(FileSystemInstrumentationFactory.class); for (FileSystemInstrumentationFactory instrumentationFactory : loader) { fs = instrumentationFactory.instrumentFileSystem(fs, broker, config); } return new ResourceInstance<>(fs); } catch (IOException | ReflectiveOperationException ioe) { throw new RuntimeException(ioe); } }
@Test public void testMultiLevelLimiter() throws Exception { SharedResourcesBrokerImpl<SimpleScopeType> broker = getBrokerForConfigMap(ImmutableMap.of( JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SimpleScopeType.GLOBAL, SharedLimiterFactory.LIMITER_CLASS_KEY), "CountBasedLimiter", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SimpleScopeType.GLOBAL, CountBasedLimiter.Factory.COUNT_KEY), "10", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SimpleScopeType.LOCAL, SharedLimiterFactory.LIMITER_CLASS_KEY), "CountBasedLimiter", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SimpleScopeType.LOCAL, CountBasedLimiter.Factory.COUNT_KEY), "5" )); SharedResourcesBroker<SimpleScopeType> localBroker1 = broker.newSubscopedBuilder(new SimpleScope<>(SimpleScopeType.LOCAL, "local1")).build(); SharedResourcesBroker<SimpleScopeType> localBroker2 = broker.newSubscopedBuilder(new SimpleScope<>(SimpleScopeType.LOCAL, "local2")).build(); SharedLimiterFactory<SimpleScopeType> factory = new SharedLimiterFactory<>(); Limiter limiter1 = ((ResourceInstance<Limiter>) factory.createResource(localBroker1, broker.getConfigView(SimpleScopeType.LOCAL, new SharedLimiterKey("resource"), factory.getName()))).getResource(); Limiter limiter2 = ((ResourceInstance<Limiter>) factory.createResource(localBroker2, broker.getConfigView(SimpleScopeType.LOCAL, new SharedLimiterKey("resource"), factory.getName()))).getResource(); Assert.assertTrue(limiter1 instanceof MultiLimiter); Assert.assertTrue(limiter2 instanceof MultiLimiter); Assert.assertEquals(((CountBasedLimiter)((MultiLimiter) limiter1).getUnderlyingLimiters().get(0)).getCountLimit(), 5); Assert.assertEquals(((CountBasedLimiter)((MultiLimiter) limiter1).getUnderlyingLimiters().get(1)).getCountLimit(), 10); Assert.assertEquals(((CountBasedLimiter)((MultiLimiter) limiter2).getUnderlyingLimiters().get(0)).getCountLimit(), 5); Assert.assertEquals(((CountBasedLimiter)((MultiLimiter) limiter2).getUnderlyingLimiters().get(1)).getCountLimit(), 10); Assert.assertNotEquals(((MultiLimiter) limiter1).getUnderlyingLimiters().get(0), ((MultiLimiter) limiter2).getUnderlyingLimiters().get(0)); Assert.assertEquals(((MultiLimiter) limiter1).getUnderlyingLimiters().get(1), ((MultiLimiter) limiter2).getUnderlyingLimiters().get(1)); }
@Override public SharedResourceFactoryResponse<BasicDataSource> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, MysqlDataSourceKey> config) throws NotConfiguredException { MysqlDataSourceKey key = config.getKey(); Config configuration = key.getConfig(); BasicDataSource dataSource = MysqlStateStore.newDataSource(configuration); return new ResourceInstance<>(dataSource); }
@Override public synchronized T getResource() { // mark the object as invalid before returning so that a new one will be created on the next // request from the factory this.valid = false; return super.getResource(); }
@Override public SharedResourceFactoryResponse<LineageInfo> createResource(SharedResourcesBroker<GobblinScopeTypes> broker, ScopedConfigView<GobblinScopeTypes, EmptyKey> config) throws NotConfiguredException { return new ResourceInstance<>(new LineageInfo(config.getConfig())); }
Object obj = ((ResourceInstance) entry.getValue()).getResource();
@Override public SharedResourceFactoryResponse<HiveConf> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, SharedHiveConfKey> config) throws NotConfiguredException { SharedHiveConfKey sharedHiveConfKey = config.getKey(); HiveConf rawConf = new HiveConf(); if (!sharedHiveConfKey.hiveConfUri.equals(SharedHiveConfKey.INSTANCE.toConfigurationKey()) && StringUtils .isNotEmpty(sharedHiveConfKey.hiveConfUri)) { rawConf.setVar(HiveConf.ConfVars.METASTOREURIS, sharedHiveConfKey.hiveConfUri); rawConf.set(HIVE_METASTORE_TOKEN_SIGNATURE, sharedHiveConfKey.hiveConfUri); } return new ResourceInstance<>(rawConf); }
@Override public SharedResourceFactoryResponse<RequestSender> createResource( SharedResourcesBroker<S> broker, ScopedConfigView<S, SharedRestClientKey> config) throws NotConfiguredException { try { List<String> connectionPrefixes = SharedRestClientFactory.parseConnectionPrefixes(config.getConfig(), config.getKey()); return new ResourceInstance<>( new RedirectAwareRestClientRequestSender(broker, connectionPrefixes)); } catch (URISyntaxException use) { throw new RuntimeException(use); } }
@Override public SharedResourceFactoryResponse<DataPublisher> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, DataPublisherKey> config) throws NotConfiguredException { try { DataPublisherKey key = config.getKey(); String publisherClassName = key.getPublisherClassName(); State state = key.getState(); Class<? extends DataPublisher> dataPublisherClass = (Class<? extends DataPublisher>) Class .forName(publisherClassName); log.info("Creating data publisher with class {} in scope {}. ", publisherClassName, config.getScope().toString()); DataPublisher publisher = DataPublisher.getInstance(dataPublisherClass, state); // If the publisher is threadsafe then it is shareable, so return it as a resource instance that may be cached // by the broker. // Otherwise, it is not shareable, so return it as an immediately invalidated resource that will only be returned // once from the broker. if (isPublisherCacheable(publisher)) { return new ResourceInstance<>(publisher); } else { return new ImmediatelyInvalidResourceEntry<>(publisher); } } catch (ReflectiveOperationException e) { throw new RuntimeException(e); } }
@Override public SharedResourceFactoryResponse<RestClient> createResource(SharedResourcesBroker<S> broker, ScopedConfigView<S, SharedRestClientKey> config) throws NotConfiguredException { try { SharedRestClientKey key = config.getKey(); if (!(key instanceof UriRestClientKey)) { return new ResourceCoordinate<>(this, new UriRestClientKey(key.serviceName, resolveUriPrefix(config.getConfig(), key)), config.getScope()); } String uriPrefix = ((UriRestClientKey) key).getUri(); HttpClientFactory http = new HttpClientFactory(FilterChains.empty(), new NioEventLoopGroup(0 /* use default settings */, ExecutorsUtils.newDaemonThreadFactory(Optional.<Logger>absent(), Optional.of("R2 Nio Event Loop-%d"))), true, Executors.newSingleThreadScheduledExecutor( ExecutorsUtils.newDaemonThreadFactory(Optional.<Logger>absent(), Optional.of("R2 Netty Scheduler"))), true); Client r2Client = new TransportClientAdapter(http.getClient(Collections.<String, String>emptyMap())); return new ResourceInstance<>(new RestClient(r2Client,uriPrefix)); } catch (URISyntaxException use) { throw new RuntimeException("Could not create a rest client for key " + Optional.fromNullable(config.getKey().toConfigurationKey()).or("null")); } }