private static HiveConf getHiveConf(Optional<String> hcatURI) { try { return HiveConfFactory.get(hcatURI, SharedResourcesBrokerFactory.getImplicitBroker()); } catch (IOException nce) { throw new RuntimeException("Implicit broker is not correctly configured, failed to fetch a HiveConf object", nce); } }
@Override public void initialize(URI uri, Configuration conf) throws IOException { this.replacementScheme = uri.getScheme(); Configuration actualConfiguration = new Configuration(conf); String key = "fs." + this.underlyingScheme + ".impl"; actualConfiguration.set(key, this.underlyingFs.getClass().getName()); this.underlyingFs = FileSystemFactory.get(replaceScheme(uri, this.replacementScheme, this.underlyingScheme), actualConfiguration, SharedResourcesBrokerFactory.getImplicitBroker()); } }
@Override public void run() { Assert.assertEquals(this.expectedBroker, SharedResourcesBrokerFactory.getImplicitBroker()); } }
@Override public DatasetStateStore<JobState.DatasetState> createStateStore(Config config) { String stateStoreTableName = config.hasPath(ConfigurationKeys.STATE_STORE_DB_TABLE_KEY) ? config.getString(ConfigurationKeys.STATE_STORE_DB_TABLE_KEY) : ConfigurationKeys.DEFAULT_STATE_STORE_DB_TABLE; boolean compressedValues = config.hasPath(ConfigurationKeys.STATE_STORE_COMPRESSED_VALUES_KEY) ? config.getBoolean(ConfigurationKeys.STATE_STORE_COMPRESSED_VALUES_KEY) : ConfigurationKeys.DEFAULT_STATE_STORE_COMPRESSED_VALUES; try { BasicDataSource basicDataSource = MysqlDataSourceFactory.get(config, SharedResourcesBrokerFactory.getImplicitBroker()); return new MysqlDatasetStateStore(basicDataSource, stateStoreTableName, compressedValues); } catch (Exception e) { throw new RuntimeException("Failed to create MysqlDatasetStateStore with factory", e); } } }
@Test public void testImplicitBroker() { Assert.assertEquals(SharedResourcesBrokerFactory.getImplicitBroker(), IMPLICIT); ExecutorService executorService = Executors.newSingleThreadExecutor(); Future<?> future = executorService.submit(new ImplicitBrokerTest()); try { future.get(); } catch (ExecutionException | InterruptedException ee) { throw new RuntimeException(ee); } executorService.shutdownNow(); }
@Override public <T extends State> StateStore<T> createStateStore(Config config, Class<T> stateClass) { String stateStoreTableName = ConfigUtils.getString(config, ConfigurationKeys.STATE_STORE_DB_TABLE_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_DB_TABLE); boolean compressedValues = ConfigUtils.getBoolean(config, ConfigurationKeys.STATE_STORE_COMPRESSED_VALUES_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_COMPRESSED_VALUES); try { BasicDataSource basicDataSource = MysqlDataSourceFactory.get(config, SharedResourcesBrokerFactory.getImplicitBroker()); return new MysqlStateStore(basicDataSource, stateStoreTableName, compressedValues, stateClass); } catch (Exception e) { throw new RuntimeException("Failed to create MysqlStateStore with factory", e); } } }
@Test public void testSameKey() throws IOException { Config config = ConfigFactory.parseMap(ImmutableMap.of(ConfigurationKeys.STATE_STORE_DB_URL_KEY, "url", ConfigurationKeys.STATE_STORE_DB_USER_KEY, "user", ConfigurationKeys.STATE_STORE_DB_PASSWORD_KEY, "dummypwd")); BasicDataSource basicDataSource1 = MysqlDataSourceFactory.get(config, SharedResourcesBrokerFactory.getImplicitBroker()); BasicDataSource basicDataSource2 = MysqlDataSourceFactory.get(config, SharedResourcesBrokerFactory.getImplicitBroker()); Assert.assertEquals(basicDataSource1, basicDataSource2); }
@Test public void testDifferentKey() throws IOException { Config config1 = ConfigFactory.parseMap(ImmutableMap.of(ConfigurationKeys.STATE_STORE_DB_URL_KEY, "url1", ConfigurationKeys.STATE_STORE_DB_USER_KEY, "user", ConfigurationKeys.STATE_STORE_DB_PASSWORD_KEY, "dummypwd")); Config config2 = ConfigFactory.parseMap(ImmutableMap.of(ConfigurationKeys.STATE_STORE_DB_URL_KEY, "url2", ConfigurationKeys.STATE_STORE_DB_USER_KEY, "user", ConfigurationKeys.STATE_STORE_DB_PASSWORD_KEY, "dummypwd")); BasicDataSource basicDataSource1 = MysqlDataSourceFactory.get(config1, SharedResourcesBrokerFactory.getImplicitBroker()); BasicDataSource basicDataSource2 = MysqlDataSourceFactory.get(config2, SharedResourcesBrokerFactory.getImplicitBroker()); Assert.assertNotEquals(basicDataSource1, basicDataSource2); }
@Override public void run() { Assert.assertEquals(SharedResourcesBrokerFactory.getImplicitBroker(), IMPLICIT); SharedResourcesBroker<SimpleScopeType> broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker( ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance()); Assert.assertNotEquals(SharedResourcesBrokerFactory.getImplicitBroker(), broker); SharedResourcesBrokerFactory.registerImplicitBroker(broker); Assert.assertEquals(SharedResourcesBrokerFactory.getImplicitBroker(), broker); ExecutorService executorService = Executors.newSingleThreadExecutor(); Future<?> future = executorService.submit(new InnerImplicitBrokerTest(broker)); try { future.get(); } catch (ExecutionException | InterruptedException ee) { throw new RuntimeException(ee); } executorService.shutdownNow(); } }
@Test public void testSameDbDifferentUser() throws IOException { Config config1 = ConfigFactory.parseMap(ImmutableMap.of(ConfigurationKeys.STATE_STORE_DB_URL_KEY, "url1", ConfigurationKeys.STATE_STORE_DB_USER_KEY, "user1", ConfigurationKeys.STATE_STORE_DB_PASSWORD_KEY, "dummypwd")); Config config2 = ConfigFactory.parseMap(ImmutableMap.of(ConfigurationKeys.STATE_STORE_DB_URL_KEY, "url1", ConfigurationKeys.STATE_STORE_DB_USER_KEY, "user2", ConfigurationKeys.STATE_STORE_DB_PASSWORD_KEY, "dummypwd")); BasicDataSource basicDataSource1 = MysqlDataSourceFactory.get(config1, SharedResourcesBrokerFactory.getImplicitBroker()); BasicDataSource basicDataSource2 = MysqlDataSourceFactory.get(config2, SharedResourcesBrokerFactory.getImplicitBroker()); Assert.assertNotEquals(basicDataSource1, basicDataSource2); } }
@Test public void testHiveConfFactory() throws Exception { HiveConf hiveConf = HiveConfFactory.get(Optional.absent(), SharedResourcesBrokerFactory.getImplicitBroker()); HiveConf hiveConf1 = HiveConfFactory.get(Optional.absent(), SharedResourcesBrokerFactory.getImplicitBroker()); Assert.assertEquals(hiveConf, hiveConf1); // When there's no hcatURI specified, the default hive-site should be loaded. Assert.assertTrue(hiveConf.getVar(METASTOREURIS).equals("file:///test")); Assert.assertTrue(hiveConf.get(HIVE_METASTORE_TOKEN_SIGNATURE).equals("file:///test")); HiveConf hiveConf2 = HiveConfFactory.get(Optional.of("hcat1"), SharedResourcesBrokerFactory.getImplicitBroker()); HiveConf hiveConf3 = HiveConfFactory.get(Optional.of("hcat1"), SharedResourcesBrokerFactory.getImplicitBroker()); Assert.assertEquals(hiveConf2, hiveConf3); HiveConf hiveConf4 = HiveConfFactory.get(Optional.of("hcat11"), SharedResourcesBrokerFactory.getImplicitBroker()); Assert.assertNotEquals(hiveConf3, hiveConf4); Assert.assertNotEquals(hiveConf4, hiveConf); // THe uri should be correctly set. Assert.assertEquals(hiveConf3.getVar(METASTOREURIS), "hcat1"); Assert.assertEquals(hiveConf3.get(HIVE_METASTORE_TOKEN_SIGNATURE), "hcat1"); Assert.assertEquals(hiveConf4.getVar(METASTOREURIS), "hcat11"); Assert.assertEquals(hiveConf4.get(HIVE_METASTORE_TOKEN_SIGNATURE), "hcat11"); } }
try { hiveConf = SharedResourcesBrokerFactory .getImplicitBroker().getSharedResource(new HiveConfFactory<>(), SharedHiveConfKey.INSTANCE); deserializer = ReflectionUtils.newInstance(hiveConf.getClassByName(serde).asSubclass(Deserializer.class), hiveConf);
private static HiveConf getHiveConf(Optional<String> hcatURI) { try { return HiveConfFactory.get(hcatURI, SharedResourcesBrokerFactory.getImplicitBroker()); } catch (IOException nce) { throw new RuntimeException("Implicit broker is not correctly configured, failed to fetch a HiveConf object", nce); } }
@Override public void initialize(URI uri, Configuration conf) throws IOException { this.replacementScheme = uri.getScheme(); Configuration actualConfiguration = new Configuration(conf); String key = "fs." + this.underlyingScheme + ".impl"; actualConfiguration.set(key, this.underlyingFs.getClass().getName()); this.underlyingFs = FileSystemFactory.get(replaceScheme(uri, this.replacementScheme, this.underlyingScheme), actualConfiguration, SharedResourcesBrokerFactory.getImplicitBroker()); } }
@Override public DatasetStateStore<JobState.DatasetState> createStateStore(Config config) { String stateStoreTableName = config.hasPath(ConfigurationKeys.STATE_STORE_DB_TABLE_KEY) ? config.getString(ConfigurationKeys.STATE_STORE_DB_TABLE_KEY) : ConfigurationKeys.DEFAULT_STATE_STORE_DB_TABLE; boolean compressedValues = config.hasPath(ConfigurationKeys.STATE_STORE_COMPRESSED_VALUES_KEY) ? config.getBoolean(ConfigurationKeys.STATE_STORE_COMPRESSED_VALUES_KEY) : ConfigurationKeys.DEFAULT_STATE_STORE_COMPRESSED_VALUES; try { BasicDataSource basicDataSource = MysqlDataSourceFactory.get(config, SharedResourcesBrokerFactory.getImplicitBroker()); return new MysqlDatasetStateStore(basicDataSource, stateStoreTableName, compressedValues); } catch (Exception e) { throw new RuntimeException("Failed to create MysqlDatasetStateStore with factory", e); } } }
@Override public <T extends State> StateStore<T> createStateStore(Config config, Class<T> stateClass) { String stateStoreTableName = ConfigUtils.getString(config, ConfigurationKeys.STATE_STORE_DB_TABLE_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_DB_TABLE); boolean compressedValues = ConfigUtils.getBoolean(config, ConfigurationKeys.STATE_STORE_COMPRESSED_VALUES_KEY, ConfigurationKeys.DEFAULT_STATE_STORE_COMPRESSED_VALUES); try { BasicDataSource basicDataSource = MysqlDataSourceFactory.get(config, SharedResourcesBrokerFactory.getImplicitBroker()); return new MysqlStateStore(basicDataSource, stateStoreTableName, compressedValues, stateClass); } catch (Exception e) { throw new RuntimeException("Failed to create MysqlStateStore with factory", e); } } }
try { hiveConf = SharedResourcesBrokerFactory .getImplicitBroker().getSharedResource(new HiveConfFactory<>(), SharedHiveConfKey.INSTANCE); deserializer = ReflectionUtils.newInstance(hiveConf.getClassByName(serde).asSubclass(Deserializer.class), hiveConf);