public SharedResourcesBroker<GobblinScopeTypes> getDefaultInstanceBroker() { getLog().warn("Creating a default instance broker for job launcher. Objects may not be shared across all jobs in this instance."); SharedResourcesBrokerImpl<GobblinScopeTypes> globalBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(getSysConfig().getConfig(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); return globalBroker.newSubscopedBuilder(new SimpleScope<>(GobblinScopeTypes.INSTANCE, getInstanceName())).build(); }
@Override public Object call() throws Exception { return factory.createResource(broker.getScopedView(scope.getType()), broker.getConfigView(scope.getType(), key, factory.getName())); } });
/** * Create a root {@link SharedResourcesBroker}. Subscoped brokers should be built using * {@link SharedResourcesBroker#newSubscopedBuilder(ScopeInstance)}. * * In general, this method will be called only once per application, and all other brokers will Nbe children of the root * application broker. * * @param config The global configuration of the broker. * @param globalScope The scope of the root broker. * @param <S> The {@link ScopeType} DAG used for this broker tree. */ public static <S extends ScopeType<S>> SharedResourcesBrokerImpl<S> createDefaultTopLevelBroker(Config config, ScopeInstance<S> globalScope) { if (!globalScope.getType().equals(globalScope.getType().rootScope())) { throw new IllegalArgumentException(String.format("The top level broker must be created at the root scope type. " + "%s is not a root scope type.", globalScope.getType())); } ScopeWrapper<S> scopeWrapper = new ScopeWrapper<>(globalScope.getType(), globalScope, Lists.<ScopeWrapper<S>>newArrayList()); return new SharedResourcesBrokerImpl<>(new DefaultBrokerCache<S>(), scopeWrapper, Lists.newArrayList(new SharedResourcesBrokerImpl.ScopedConfig<>(globalScope.getType(), getBrokerConfig(addSystemConfigurationToConfig(config)))), ImmutableMap.of(globalScope.getType(), scopeWrapper)); }
@Override public ScopeWrapper<S> call() throws Exception { return broker.getWrappedScope(factory.getAutoScope(broker, broker.getConfigView(null, key, factory.getName()))); } });
@Test public void testScoping() throws Exception { // Correct creation behavior Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); Assert.assertEquals(jobBroker.getScope(GobblinScopeTypes.INSTANCE).getType(), GobblinScopeTypes.INSTANCE); Assert.assertEquals(jobBroker.getScope(GobblinScopeTypes.INSTANCE).getClass(), GobblinScopeInstance.class); Assert.assertEquals(jobBroker.getScope(GobblinScopeTypes.INSTANCE), GobblinScopeTypes.INSTANCE.defaultScopeInstance()); Assert.assertEquals(jobBroker.getScope(GobblinScopeTypes.JOB).getType(), GobblinScopeTypes.JOB); Assert.assertEquals(jobBroker.getScope(GobblinScopeTypes.JOB).getClass(), JobScopeInstance.class); Assert.assertEquals(((JobScopeInstance) jobBroker.getScope(GobblinScopeTypes.JOB)).getJobId(), "job123"); try { jobBroker.getScope(GobblinScopeTypes.TASK); Assert.fail(); } catch (NoSuchScopeException nsse) { // should throw no scope exception } }
@Test public void testScopedView() throws Exception { Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> instanceView = jobBroker.getScopedView(GobblinScopeTypes.INSTANCE); Assert.assertEquals(instanceView.selfScope().getType(), GobblinScopeTypes.INSTANCE); TestFactory.SharedResource resource = instanceView.getSharedResource(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey")); TestFactory.SharedResource resource2 = jobBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.INSTANCE); Assert.assertEquals(resource, resource2); try { instanceView.newSubscopedBuilder(new JobScopeInstance("otherJob", "job234")); Assert.fail(); } catch (UnsupportedOperationException exc) { // Expected } }
topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")). withOverridingConfig(overrides).build(); jobBroker.getConfigView(GobblinScopeTypes.CONTAINER, new TestResourceKey(key), TestFactory.NAME); Assert.assertEquals(configView.getConfig().getString("key1"), "value1"); Assert.assertEquals(configView.getConfig().getString("key2"), "value2scopekey"); jobBroker.getConfigView(GobblinScopeTypes.JOB, new TestResourceKey(key), TestFactory.NAME); Assert.assertEquals(configView.getConfig().getString("key1"), "value1"); Assert.assertEquals(configView.getConfig().getString("key2"), "value2scopekey_o");
@Test public void testAutoscoping() throws Exception { Config config = ConfigFactory.empty(); config = TestFactory.setAutoScopeLevel(config, GobblinScopeTypes.JOB); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> containerBroker = topBroker.newSubscopedBuilder(GobblinScopeTypes.CONTAINER.defaultScopeInstance()).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> taskBroker = jobBroker.newSubscopedBuilder(new TaskScopeInstance("taskabc")) .withAdditionalParentBroker(containerBroker).build(); TestFactory.SharedResource jobScopedResource = taskBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.JOB); TestFactory.SharedResource taskScopedResource = taskBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.TASK); TestFactory.SharedResource autoscopedResource = taskBroker.getSharedResource(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey")); Assert.assertEquals(jobScopedResource, autoscopedResource); Assert.assertNotEquals(taskScopedResource, autoscopedResource); }
@Test public void testCreationOfBrokers() throws Exception { // Correct creation behavior Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, new SimpleScope<GobblinScopeTypes>(GobblinScopeTypes.GLOBAL, "myGlobalScope")); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> containerBroker = topBroker.newSubscopedBuilder(GobblinScopeTypes.CONTAINER.defaultScopeInstance()).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> taskBroker = jobBroker.newSubscopedBuilder(new TaskScopeInstance("taskabc")) .withAdditionalParentBroker(containerBroker).build(); Assert.assertEquals(taskBroker.selfScope().getType(), GobblinScopeTypes.TASK); Assert.assertEquals(((TaskScopeInstance) taskBroker.selfScope()).getTaskId(), "taskabc"); Assert.assertEquals(taskBroker.getScope(GobblinScopeTypes.CONTAINER).getType(), GobblinScopeTypes.CONTAINER); Assert.assertEquals(((GobblinScopeInstance) taskBroker.getScope(GobblinScopeTypes.CONTAINER)).getScopeId(), "container"); }
@Test public void testAutoscopedResourcesOnlyClosedInCorrectScope() throws Exception { Config config = ConfigFactory.empty(); config = TestFactory.setAutoScopeLevel(config, GobblinScopeTypes.JOB); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> containerBroker = topBroker.newSubscopedBuilder(GobblinScopeTypes.CONTAINER.defaultScopeInstance()).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> taskBroker = jobBroker.newSubscopedBuilder(new TaskScopeInstance("taskabc")) .withAdditionalParentBroker(containerBroker).build(); TestFactory.SharedResource autoscopedResource = taskBroker.getSharedResource(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey")); // since object autoscopes at job level, it should not be closed if we close the task broker taskBroker.close(); Assert.assertFalse(autoscopedResource.isClosed()); // however, when closing job broker, resource should be closed jobBroker.close(); Assert.assertTrue(autoscopedResource.isClosed()); }
@Test public void testLifecycle() throws Exception { Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> containerBroker = topBroker.newSubscopedBuilder(GobblinScopeTypes.CONTAINER.defaultScopeInstance()).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> taskBroker = jobBroker.newSubscopedBuilder(new TaskScopeInstance("taskabc")) .withAdditionalParentBroker(containerBroker).build(); // create a shared resource TestFactory.SharedResource jobResource = taskBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.JOB); TestFactory.SharedResource taskResource = taskBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.TASK); Assert.assertFalse(jobResource.isClosed()); Assert.assertFalse(taskResource.isClosed()); taskBroker.close(); // only resources at lower scopes than task should be closed Assert.assertFalse(jobResource.isClosed()); Assert.assertTrue(taskResource.isClosed()); // since taskResource has been closed, broker should return a new instance of the object TestFactory.SharedResource taskResource2 = taskBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.TASK); Assert.assertNotEquals(taskResource, taskResource2); topBroker.close(); Assert.assertTrue(jobResource.isClosed()); Assert.assertTrue(taskResource.isClosed()); }
@Test public void testRedirect() throws Exception { Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); // create a shared resource TestFactory.SharedResource resource = jobBroker.getSharedResourceAtScope(new TestFactoryWithRedirect<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.JOB); Assert.assertEquals(resource.getKey(), "myKey"); Assert.assertEquals(topBroker.getSharedResourceAtScope(new TestFactoryWithRedirect<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.GLOBAL), resource); Assert.assertEquals(topBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.GLOBAL), resource); }
@Test public void test() throws Exception { SharedResourcesBrokerImpl<SimpleScopeType> broker = SharedResourcesBrokerFactory.<SimpleScopeType>createDefaultTopLevelBroker( ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance()); FileSystemKey key = new FileSystemKey(new URI("file:///"), new Configuration()); FileSystemFactory<SimpleScopeType> factory = new FileSystemFactory<>(); FileSystem fs = broker.getSharedResource(factory, key); verifyInstrumentedOnce(fs); SharedResourcesBroker<SimpleScopeType> subBroker = broker.newSubscopedBuilder(SimpleScopeType.LOCAL.defaultScopeInstance()).build(); FileSystem subBrokerFs = FileSystemFactory.get(new URI("file:///"), new Configuration(), subBroker); Assert.assertEquals(fs, subBrokerFs); }
@Override public <T, K extends SharedResourceKey> T getSharedResourceAtScope(SharedResourceFactory<T, K, S> factory, K key, S scope) throws NotConfiguredException, NoSuchScopeException { try { return this.brokerCache.getScoped(factory, key, getWrappedScope(scope), this); } catch (ExecutionException ee) { throw new RuntimeException(ee); } }
@Test public void testLoadingOfClasspath() { Config config = ConfigFactory.parseMap(ImmutableMap.of(SharedResourcesBrokerFactory.BROKER_CONF_FILE_KEY, "/broker/testBroker.conf")); SharedResourcesBrokerImpl<SimpleScopeType> broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, SimpleScopeType.GLOBAL.defaultScopeInstance()); ConfigView configView = broker.getConfigView(null, null, "factory"); Assert.assertTrue(configView.getConfig().hasPath("testKey")); Assert.assertEquals(configView.getConfig().getString("testKey"), "testValue"); }
/** * Get an object for the specified factory, key, and broker at the scope selected by the factory. {@link DefaultBrokerCache} * guarantees that calling this method from brokers with the same leaf scope will return the same object. */ @SuppressWarnings(value = "unchecked") <T, K extends SharedResourceKey> T getAutoScoped(final SharedResourceFactory<T, K, S> factory, final K key, final SharedResourcesBrokerImpl<S> broker) throws ExecutionException { // figure out auto scope RawJobBrokerKey autoscopeCacheKey = new RawJobBrokerKey(broker.getWrappedSelfScope(), factory.getName(), key); ScopeWrapper<S> selectedScope = this.autoScopeCache.get(autoscopeCacheKey, new Callable<ScopeWrapper<S>>() { @Override public ScopeWrapper<S> call() throws Exception { return broker.getWrappedScope(factory.getAutoScope(broker, broker.getConfigView(null, key, factory.getName()))); } }); // get actual object return getScoped(factory, key, selectedScope, broker); }
@Test public void testMultiLevelLimiter() throws Exception { SharedResourcesBrokerImpl<SimpleScopeType> broker = getBrokerForConfigMap(ImmutableMap.of( JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SimpleScopeType.GLOBAL, SharedLimiterFactory.LIMITER_CLASS_KEY), "CountBasedLimiter", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SimpleScopeType.GLOBAL, CountBasedLimiter.Factory.COUNT_KEY), "10", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SimpleScopeType.LOCAL, SharedLimiterFactory.LIMITER_CLASS_KEY), "CountBasedLimiter", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SimpleScopeType.LOCAL, CountBasedLimiter.Factory.COUNT_KEY), "5" )); SharedResourcesBroker<SimpleScopeType> localBroker1 = broker.newSubscopedBuilder(new SimpleScope<>(SimpleScopeType.LOCAL, "local1")).build(); SharedResourcesBroker<SimpleScopeType> localBroker2 = broker.newSubscopedBuilder(new SimpleScope<>(SimpleScopeType.LOCAL, "local2")).build(); SharedLimiterFactory<SimpleScopeType> factory = new SharedLimiterFactory<>(); Limiter limiter1 = ((ResourceInstance<Limiter>) factory.createResource(localBroker1, broker.getConfigView(SimpleScopeType.LOCAL, new SharedLimiterKey("resource"), factory.getName()))).getResource(); Limiter limiter2 = ((ResourceInstance<Limiter>) factory.createResource(localBroker2, broker.getConfigView(SimpleScopeType.LOCAL, new SharedLimiterKey("resource"), factory.getName()))).getResource(); Assert.assertTrue(limiter1 instanceof MultiLimiter); Assert.assertTrue(limiter2 instanceof MultiLimiter); Assert.assertEquals(((CountBasedLimiter)((MultiLimiter) limiter1).getUnderlyingLimiters().get(0)).getCountLimit(), 5); Assert.assertEquals(((CountBasedLimiter)((MultiLimiter) limiter1).getUnderlyingLimiters().get(1)).getCountLimit(), 10); Assert.assertEquals(((CountBasedLimiter)((MultiLimiter) limiter2).getUnderlyingLimiters().get(0)).getCountLimit(), 5); Assert.assertEquals(((CountBasedLimiter)((MultiLimiter) limiter2).getUnderlyingLimiters().get(1)).getCountLimit(), 10); Assert.assertNotEquals(((MultiLimiter) limiter1).getUnderlyingLimiters().get(0), ((MultiLimiter) limiter2).getUnderlyingLimiters().get(0)); Assert.assertEquals(((MultiLimiter) limiter1).getUnderlyingLimiters().get(1), ((MultiLimiter) limiter2).getUnderlyingLimiters().get(1)); }
GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> containerBroker = topBroker.newSubscopedBuilder(GobblinScopeTypes.CONTAINER.defaultScopeInstance()).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> taskBroker = jobBroker.newSubscopedBuilder(new TaskScopeInstance("taskabc")) .withAdditionalParentBroker(containerBroker).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> taskBroker2 = jobBroker.newSubscopedBuilder(new TaskScopeInstance("taskxyz")) .withAdditionalParentBroker(containerBroker).build(); taskBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.JOB); Assert.assertEquals(taskBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.JOB), resource); Assert.assertEquals(taskBroker2.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.JOB), resource); Assert.assertEquals(jobBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.JOB), resource); Assert.assertNotEquals(taskBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("otherKey"), GobblinScopeTypes.JOB), resource); Assert.assertNotEquals(taskBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.TASK), resource); Assert.assertNotEquals(taskBroker.getSharedResource(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey")), resource);
@Test public void testConfigurationInjection() throws Exception { String key = "myKey"; Config config = ConfigFactory.parseMap(ImmutableMap.of( JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, "key1"), "value1", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, "key2"), "value2", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.CONTAINER.name(), "key2"), "value2scope", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, key, "key2"), "value2key", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.CONTAINER.name(), key, "key2"), "value2scopekey" )); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> containerBroker = topBroker.newSubscopedBuilder(GobblinScopeTypes.CONTAINER.defaultScopeInstance()).build(); // create a shared resource TestFactory.SharedResource resource = containerBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.CONTAINER); Assert.assertEquals(resource.getConfig().getString("key1"), "value1"); Assert.assertEquals(resource.getConfig().getString("key2"), "value2scopekey"); }
@Override public ScopeWrapper<S> call() throws Exception { return broker.getWrappedScope(factory.getAutoScope(broker, broker.getConfigView(null, key, factory.getName()))); } });