@Test public void testExpiringResource() throws Exception { Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<SimpleScopeType> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, SimpleScopeType.GLOBAL.defaultScopeInstance()); InvalidatableResourceFactory factory = new InvalidatableResourceFactory(); long value = topBroker.getSharedResource(factory, new EmptyKey()); Assert.assertEquals(topBroker.getSharedResource(factory, new EmptyKey()), new Long(value)); factory.getLastResourceEntry().setValid(false); Assert.assertNotEquals(topBroker.getSharedResource(factory, new EmptyKey()), value); value = topBroker.getSharedResource(factory, new EmptyKey()); Assert.assertEquals(topBroker.getSharedResource(factory, new EmptyKey()), new Long(value)); }
resource); Assert.assertNotEquals(taskBroker.getSharedResource(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey")), resource);
@Test public void testScopedView() throws Exception { Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> instanceView = jobBroker.getScopedView(GobblinScopeTypes.INSTANCE); Assert.assertEquals(instanceView.selfScope().getType(), GobblinScopeTypes.INSTANCE); TestFactory.SharedResource resource = instanceView.getSharedResource(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey")); TestFactory.SharedResource resource2 = jobBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.INSTANCE); Assert.assertEquals(resource, resource2); try { instanceView.newSubscopedBuilder(new JobScopeInstance("otherJob", "job234")); Assert.fail(); } catch (UnsupportedOperationException exc) { // Expected } }
@Test public void testCreationWithConfigurationFSImpl() throws Exception { SharedResourcesBrokerImpl<SimpleScopeType> broker = SharedResourcesBrokerFactory.<SimpleScopeType>createDefaultTopLevelBroker( ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance()); Configuration conf = new Configuration(); conf.set("fs.local.impl", InstrumentedLocalFileSystem.class.getName()); FileSystemKey key = new FileSystemKey(new URI("file:///"), new Configuration()); FileSystemFactory<SimpleScopeType> factory = new FileSystemFactory<>(); FileSystem fs = broker.getSharedResource(factory, key); verifyInstrumentedOnce(fs); Assert.assertTrue(DecoratorUtils.resolveUnderlyingObject(fs) instanceof LocalFileSystem); }
@Test public void testAutoscoping() throws Exception { Config config = ConfigFactory.empty(); config = TestFactory.setAutoScopeLevel(config, GobblinScopeTypes.JOB); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> containerBroker = topBroker.newSubscopedBuilder(GobblinScopeTypes.CONTAINER.defaultScopeInstance()).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> taskBroker = jobBroker.newSubscopedBuilder(new TaskScopeInstance("taskabc")) .withAdditionalParentBroker(containerBroker).build(); TestFactory.SharedResource jobScopedResource = taskBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.JOB); TestFactory.SharedResource taskScopedResource = taskBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.TASK); TestFactory.SharedResource autoscopedResource = taskBroker.getSharedResource(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey")); Assert.assertEquals(jobScopedResource, autoscopedResource); Assert.assertNotEquals(taskScopedResource, autoscopedResource); }
@Test public void testCreationWithInstrumentedScheme() throws Exception { SharedResourcesBrokerImpl<SimpleScopeType> broker = SharedResourcesBrokerFactory.<SimpleScopeType>createDefaultTopLevelBroker( ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance()); FileSystemKey key = new FileSystemKey(new URI("instrumented-file:///"), new Configuration()); FileSystemFactory<SimpleScopeType> factory = new FileSystemFactory<>(); FileSystem fs = broker.getSharedResource(factory, key); verifyInstrumentedOnce(fs); Assert.assertTrue(DecoratorUtils.resolveUnderlyingObject(fs) instanceof LocalFileSystem); }
@Test public void test() throws Exception { SharedResourcesBrokerImpl<SimpleScopeType> broker = SharedResourcesBrokerFactory.<SimpleScopeType>createDefaultTopLevelBroker( ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance()); FileSystemKey key = new FileSystemKey(new URI("file:///"), new Configuration()); FileSystemFactory<SimpleScopeType> factory = new FileSystemFactory<>(); FileSystem fs = broker.getSharedResource(factory, key); verifyInstrumentedOnce(fs); SharedResourcesBroker<SimpleScopeType> subBroker = broker.newSubscopedBuilder(SimpleScopeType.LOCAL.defaultScopeInstance()).build(); FileSystem subBrokerFs = FileSystemFactory.get(new URI("file:///"), new Configuration(), subBroker); Assert.assertEquals(fs, subBrokerFs); }
@Test public void testExplicitBinding() throws Exception { Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<SimpleScopeType> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, SimpleScopeType.GLOBAL.defaultScopeInstance()); SharedResourceFactory<Long, EmptyKey, SimpleScopeType> factory = new SharedResourceFactory<Long, EmptyKey, SimpleScopeType>() { @Override public String getName() { return "myTestFactory"; } @Override public SharedResourceFactoryResponse<Long> createResource(SharedResourcesBroker<SimpleScopeType> broker, ScopedConfigView<SimpleScopeType, EmptyKey> config) throws NotConfiguredException { throw new UnsupportedOperationException(); } @Override public SimpleScopeType getAutoScope(SharedResourcesBroker<SimpleScopeType> broker, ConfigView<SimpleScopeType, EmptyKey> config) { return broker.selfScope().getType(); } }; topBroker.bindSharedResourceAtScope(factory, new EmptyKey(), SimpleScopeType.GLOBAL, 10l); Assert.assertEquals(topBroker.getSharedResource(factory, new EmptyKey()), new Long(10)); }
@Test public void testAutoscopedResourcesOnlyClosedInCorrectScope() throws Exception { Config config = ConfigFactory.empty(); config = TestFactory.setAutoScopeLevel(config, GobblinScopeTypes.JOB); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> containerBroker = topBroker.newSubscopedBuilder(GobblinScopeTypes.CONTAINER.defaultScopeInstance()).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> taskBroker = jobBroker.newSubscopedBuilder(new TaskScopeInstance("taskabc")) .withAdditionalParentBroker(containerBroker).build(); TestFactory.SharedResource autoscopedResource = taskBroker.getSharedResource(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey")); // since object autoscopes at job level, it should not be closed if we close the task broker taskBroker.close(); Assert.assertFalse(autoscopedResource.isClosed()); // however, when closing job broker, resource should be closed jobBroker.close(); Assert.assertTrue(autoscopedResource.isClosed()); }