private static SharedResourcesBroker<GobblinScopeTypes> createDefaultInstanceBroker(Properties jobProps) { LOG.warn("Creating a job specific {}. Objects will only be shared at the job level.", SharedResourcesBroker.class.getSimpleName()); return SharedResourcesBrokerFactory.createDefaultTopLevelBroker(ConfigFactory.parseProperties(jobProps), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); }
private static HiveConf getHiveConf(Optional<String> hcatURI) { try { return HiveConfFactory.get(hcatURI, SharedResourcesBrokerFactory.getImplicitBroker()); } catch (IOException nce) { throw new RuntimeException("Implicit broker is not correctly configured, failed to fetch a HiveConf object", nce); } }
@Override protected void setup(Context context) throws IOException, InterruptedException { Map<String, String> configMap = Maps.newHashMap(); SharedResourcesBrokerFactory.addBrokerKeys(configMap, context.getConfiguration()); this.broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(ConfigFactory.parseMap(configMap), SimpleScopeType.GLOBAL.defaultScopeInstance()); super.setup(context); }
@Override public void run() { Assert.assertEquals(SharedResourcesBrokerFactory.getImplicitBroker(), IMPLICIT); SharedResourcesBroker<SimpleScopeType> broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker( ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance()); Assert.assertNotEquals(SharedResourcesBrokerFactory.getImplicitBroker(), broker); SharedResourcesBrokerFactory.registerImplicitBroker(broker); Assert.assertEquals(SharedResourcesBrokerFactory.getImplicitBroker(), broker); ExecutorService executorService = Executors.newSingleThreadExecutor(); Future<?> future = executorService.submit(new InnerImplicitBrokerTest(broker)); try { future.get(); } catch (ExecutionException | InterruptedException ee) { throw new RuntimeException(ee); } executorService.shutdownNow(); } }
/** * Create a root {@link SharedResourcesBroker}. Subscoped brokers should be built using * {@link SharedResourcesBroker#newSubscopedBuilder(ScopeInstance)}. * * In general, this method will be called only once per application, and all other brokers will Nbe children of the root * application broker. * * @param config The global configuration of the broker. * @param globalScope The scope of the root broker. * @param <S> The {@link ScopeType} DAG used for this broker tree. */ public static <S extends ScopeType<S>> SharedResourcesBrokerImpl<S> createDefaultTopLevelBroker(Config config, ScopeInstance<S> globalScope) { if (!globalScope.getType().equals(globalScope.getType().rootScope())) { throw new IllegalArgumentException(String.format("The top level broker must be created at the root scope type. " + "%s is not a root scope type.", globalScope.getType())); } ScopeWrapper<S> scopeWrapper = new ScopeWrapper<>(globalScope.getType(), globalScope, Lists.<ScopeWrapper<S>>newArrayList()); return new SharedResourcesBrokerImpl<>(new DefaultBrokerCache<S>(), scopeWrapper, Lists.newArrayList(new SharedResourcesBrokerImpl.ScopedConfig<>(globalScope.getType(), getBrokerConfig(addSystemConfigurationToConfig(config)))), ImmutableMap.of(globalScope.getType(), scopeWrapper)); }
private static Config addSystemConfigurationToConfig(Config config) { Map<String, String> confMap = Maps.newHashMap(); addBrokerKeys(confMap, System.getenv().entrySet()); addBrokerKeys(confMap, System.getProperties().entrySet()); Config systemConfig = ConfigFactory.parseMap(confMap); Config tmpConfig = config.withFallback(systemConfig); String brokerConfPath = DEFAULT_BROKER_CONF_FILE; if (tmpConfig.hasPath(BROKER_CONF_FILE_KEY)) { brokerConfPath = tmpConfig.getString(BROKER_CONF_FILE_KEY); } Config resourceConfig = ConfigFactory.parseResources(SharedResourcesBrokerFactory.class, brokerConfPath); config = config.withFallback(resourceConfig).withFallback(systemConfig); if (ConfigUtils.getBoolean(config, LOAD_HADOOP_CONFIGURATION, true)) { Map<String, String> hadoopConfMap = Maps.newHashMap(); Configuration hadoopConf = new Configuration(); hadoopConf.addResource("gobblin-site.xml"); addBrokerKeys(hadoopConfMap, hadoopConf); config = config.withFallback(ConfigFactory.parseMap(hadoopConfMap)); } return config; }
@Test public void testGetBrokerConfig() { Map<String, String> srcConfigMap = new HashMap<>(); srcConfigMap.put("gobblin.broker.key1", "value1"); // Test global namespace, "gobblin.broker" Config brokerConfig = SharedResourcesBrokerFactory.getBrokerConfig(ConfigFactory.parseMap(srcConfigMap)); Config expectedConfig = ConfigFactory.parseMap(ImmutableMap.of("key1", "value1")); Assert.assertEquals(brokerConfig, expectedConfig); // Test extra namespace, "gobblin.shared" srcConfigMap.put("gobblin.shared.key2", "value2"); srcConfigMap.put("gobblin.brokerNamespaces", "gobblin.shared"); brokerConfig = SharedResourcesBrokerFactory.getBrokerConfig(ConfigFactory.parseMap(srcConfigMap)); expectedConfig = ConfigFactory.parseMap(ImmutableMap.of("key1", "value1","key2", "value2")); Assert.assertEquals(brokerConfig, expectedConfig); // Test a list of extra namespaces, configurations are respected in order srcConfigMap.put("gobblin.shared.key2", "value2"); srcConfigMap.put("gobblin.shared.key3", "value3"); srcConfigMap.put("gobblin.shared2.key3", "value3x"); srcConfigMap.put("gobblin.shared2.key4", "value4"); srcConfigMap.put("gobblin.brokerNamespaces", "gobblin.shared, gobblin.shared2"); brokerConfig = SharedResourcesBrokerFactory.getBrokerConfig(ConfigFactory.parseMap(srcConfigMap)); expectedConfig = ConfigFactory.parseMap(ImmutableMap.of("key1", "value1", "key2", "value2", "key3", "value3", "key4", "value4")); Assert.assertEquals(brokerConfig, expectedConfig); }
/** * Create a root {@link SharedResourcesBroker}. Subscoped brokers should be built using * {@link SharedResourcesBroker#newSubscopedBuilder(ScopeInstance)}. * * In general, this method will be called only once per application, and all other brokers will Nbe children of the root * application broker. * * @param config The global configuration of the broker. * @param globalScope The scope of the root broker. * @param <S> The {@link ScopeType} DAG used for this broker tree. */ public static <S extends ScopeType<S>> SharedResourcesBrokerImpl<S> createDefaultTopLevelBroker(Config config, ScopeInstance<S> globalScope) { if (!globalScope.getType().equals(globalScope.getType().rootScope())) { throw new IllegalArgumentException(String.format("The top level broker must be created at the root scope type. " + "%s is not a root scope type.", globalScope.getType())); } ScopeWrapper<S> scopeWrapper = new ScopeWrapper<>(globalScope.getType(), globalScope, Lists.<ScopeWrapper<S>>newArrayList()); return new SharedResourcesBrokerImpl<>(new DefaultBrokerCache<S>(), scopeWrapper, Lists.newArrayList(new SharedResourcesBrokerImpl.ScopedConfig<>(globalScope.getType(), getBrokerConfig(addSystemConfigurationToConfig(config)))), ImmutableMap.of(globalScope.getType(), scopeWrapper)); }
private static Config addSystemConfigurationToConfig(Config config) { Map<String, String> confMap = Maps.newHashMap(); addBrokerKeys(confMap, System.getenv().entrySet()); addBrokerKeys(confMap, System.getProperties().entrySet()); Config systemConfig = ConfigFactory.parseMap(confMap); Config tmpConfig = config.withFallback(systemConfig); String brokerConfPath = DEFAULT_BROKER_CONF_FILE; if (tmpConfig.hasPath(BROKER_CONF_FILE_KEY)) { brokerConfPath = tmpConfig.getString(BROKER_CONF_FILE_KEY); } Config resourceConfig = ConfigFactory.parseResources(SharedResourcesBrokerFactory.class, brokerConfPath); config = config.withFallback(resourceConfig).withFallback(systemConfig); if (ConfigUtils.getBoolean(config, LOAD_HADOOP_CONFIGURATION, true)) { Map<String, String> hadoopConfMap = Maps.newHashMap(); Configuration hadoopConf = new Configuration(); hadoopConf.addResource("gobblin-site.xml"); addBrokerKeys(hadoopConfMap, hadoopConf); config = config.withFallback(ConfigFactory.parseMap(hadoopConfMap)); } return config; }
private Injector createInjector(final Config config, final Optional<LeaderFinder<URIMetadata>> leaderFinder) { final SharedResourcesBroker<ThrottlingServerScopes> topLevelBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, ThrottlingServerScopes.GLOBAL.defaultScopeInstance());
@Override public void initialize(URI uri, Configuration conf) throws IOException { this.replacementScheme = uri.getScheme(); Configuration actualConfiguration = new Configuration(conf); String key = "fs." + this.underlyingScheme + ".impl"; actualConfiguration.set(key, this.underlyingFs.getClass().getName()); this.underlyingFs = FileSystemFactory.get(replaceScheme(uri, this.replacementScheme, this.underlyingScheme), actualConfiguration, SharedResourcesBrokerFactory.getImplicitBroker()); } }
private static synchronized SharedResourcesBroker<SimpleScopeType> getSingleton() { if (SINGLETON == null) { SINGLETON = createDefaultTopLevelBroker(ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance()); } return SINGLETON; }
@Override public void run() { Assert.assertEquals(this.expectedBroker, SharedResourcesBrokerFactory.getImplicitBroker()); } }
private static SharedResourcesBroker createMockBroker() { Joiner JOINER = Joiner.on("."); Config config = ConfigFactory.parseMap(ImmutableMap.of( JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SharedLimiterFactory.LIMITER_CLASS_KEY), "qps", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, RateBasedLimiter.Factory.QPS_KEY), "10" )); SharedResourcesBrokerImpl broker = SharedResourcesBrokerFactory.<SimpleScopeType>createDefaultTopLevelBroker(config, SimpleScopeType.GLOBAL.defaultScopeInstance()); return broker; }
@Override public DatasetStateStore<JobState.DatasetState> createStateStore(Config config) { String stateStoreTableName = config.hasPath(ConfigurationKeys.STATE_STORE_DB_TABLE_KEY) ? config.getString(ConfigurationKeys.STATE_STORE_DB_TABLE_KEY) : ConfigurationKeys.DEFAULT_STATE_STORE_DB_TABLE; boolean compressedValues = config.hasPath(ConfigurationKeys.STATE_STORE_COMPRESSED_VALUES_KEY) ? config.getBoolean(ConfigurationKeys.STATE_STORE_COMPRESSED_VALUES_KEY) : ConfigurationKeys.DEFAULT_STATE_STORE_COMPRESSED_VALUES; try { BasicDataSource basicDataSource = MysqlDataSourceFactory.get(config, SharedResourcesBrokerFactory.getImplicitBroker()); return new MysqlDatasetStateStore(basicDataSource, stateStoreTableName, compressedValues); } catch (Exception e) { throw new RuntimeException("Failed to create MysqlDatasetStateStore with factory", e); } } }
public void run() throws IOException, InterruptedException { List<WorkUnit> workUnits = getWorkUnits(); JobState jobState = getJobState(); Config jobConfig = getConfigFromJobState(jobState); _logger.debug("SingleTask.run: jobId {} workUnitFilePath {} jobStateFilePath {} jobState {} jobConfig {}", _jobId, _workUnitFilePath, _jobStateFilePath, jobState, jobConfig); try (SharedResourcesBroker<GobblinScopeTypes> globalBroker = SharedResourcesBrokerFactory .createDefaultTopLevelBroker(jobConfig, GobblinScopeTypes.GLOBAL.defaultScopeInstance())) { SharedResourcesBroker<GobblinScopeTypes> jobBroker = getJobBroker(jobState, globalBroker); _taskattempt = _taskAttemptBuilder.build(workUnits.iterator(), _jobId, jobState, jobBroker); _taskattempt.runAndOptionallyCommitTaskAttempt(GobblinMultiTaskAttempt.CommitPolicy.IMMEDIATE); } }
@Test public void testImplicitBroker() { Assert.assertEquals(SharedResourcesBrokerFactory.getImplicitBroker(), IMPLICIT); ExecutorService executorService = Executors.newSingleThreadExecutor(); Future<?> future = executorService.submit(new ImplicitBrokerTest()); try { future.get(); } catch (ExecutionException | InterruptedException ee) { throw new RuntimeException(ee); } executorService.shutdownNow(); }
@Test() public void testMultiThreadedGetNonThreadSafePublisher() throws InterruptedException, ExecutionException, IOException { SharedResourcesBroker broker = SharedResourcesBrokerFactory.<SimpleScopeType>createDefaultTopLevelBroker(ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance()); ExecutorService service = Executors.newFixedThreadPool(40); List<Future<?>> futures = new ArrayList<>(); for (int i = 0; i < 100000; i++) { futures.add(service.submit(new GetNonThreadSafePublisher(broker))); } for (Future f: futures) { f.get(); } service.shutdown(); service.awaitTermination(100, TimeUnit.SECONDS); }
@Override public <T extends State> StateStore<T> createStateStore(Config config, Class<T> stateClass) { String stateStoreTableName = ConfigUtils.getString(config, ConfigurationKeys.STATE_STORE_DB_TABLE_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_DB_TABLE); boolean compressedValues = ConfigUtils.getBoolean(config, ConfigurationKeys.STATE_STORE_COMPRESSED_VALUES_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_COMPRESSED_VALUES); try { BasicDataSource basicDataSource = MysqlDataSourceFactory.get(config, SharedResourcesBrokerFactory.getImplicitBroker()); return new MysqlStateStore(basicDataSource, stateStoreTableName, compressedValues, stateClass); } catch (Exception e) { throw new RuntimeException("Failed to create MysqlStateStore with factory", e); } } }
private SharedResourcesBrokerImpl<SimpleScopeType> getBrokerForConfigMap(Map<String, String> configMap) { Config config = ConfigFactory.parseMap(configMap); return SharedResourcesBrokerFactory.<SimpleScopeType>createDefaultTopLevelBroker(config, SimpleScopeType.GLOBAL.defaultScopeInstance()); }