private static SharedResourcesBroker<GobblinScopeTypes> createDefaultInstanceBroker(Properties jobProps) { LOG.warn("Creating a job specific {}. Objects will only be shared at the job level.", SharedResourcesBroker.class.getSimpleName()); return SharedResourcesBrokerFactory.createDefaultTopLevelBroker(ConfigFactory.parseProperties(jobProps), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); }
private static synchronized SharedResourcesBroker<SimpleScopeType> getSingleton() { if (SINGLETON == null) { SINGLETON = createDefaultTopLevelBroker(ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance()); } return SINGLETON; }
@Override protected void setup(Context context) throws IOException, InterruptedException { Map<String, String> configMap = Maps.newHashMap(); SharedResourcesBrokerFactory.addBrokerKeys(configMap, context.getConfiguration()); this.broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(ConfigFactory.parseMap(configMap), SimpleScopeType.GLOBAL.defaultScopeInstance()); super.setup(context); }
private static SharedResourcesBroker createMockBroker() { Joiner JOINER = Joiner.on("."); Config config = ConfigFactory.parseMap(ImmutableMap.of( JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, SharedLimiterFactory.LIMITER_CLASS_KEY), "qps", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, SharedLimiterFactory.NAME, RateBasedLimiter.Factory.QPS_KEY), "10" )); SharedResourcesBrokerImpl broker = SharedResourcesBrokerFactory.<SimpleScopeType>createDefaultTopLevelBroker(config, SimpleScopeType.GLOBAL.defaultScopeInstance()); return broker; }
@Test public void testLoadingOfClasspath() { Config config = ConfigFactory.parseMap(ImmutableMap.of(SharedResourcesBrokerFactory.BROKER_CONF_FILE_KEY, "/broker/testBroker.conf")); SharedResourcesBrokerImpl<SimpleScopeType> broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, SimpleScopeType.GLOBAL.defaultScopeInstance()); ConfigView configView = broker.getConfigView(null, null, "factory"); Assert.assertTrue(configView.getConfig().hasPath("testKey")); Assert.assertEquals(configView.getConfig().getString("testKey"), "testValue"); }
private SharedResourcesBrokerImpl<SimpleScopeType> getBrokerForConfigMap(Map<String, String> configMap) { Config config = ConfigFactory.parseMap(configMap); return SharedResourcesBrokerFactory.<SimpleScopeType>createDefaultTopLevelBroker(config, SimpleScopeType.GLOBAL.defaultScopeInstance()); }
@Test public void testCreationWithConfigurationFSImpl() throws Exception { SharedResourcesBrokerImpl<SimpleScopeType> broker = SharedResourcesBrokerFactory.<SimpleScopeType>createDefaultTopLevelBroker( ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance()); Configuration conf = new Configuration(); conf.set("fs.local.impl", InstrumentedLocalFileSystem.class.getName()); FileSystemKey key = new FileSystemKey(new URI("file:///"), new Configuration()); FileSystemFactory<SimpleScopeType> factory = new FileSystemFactory<>(); FileSystem fs = broker.getSharedResource(factory, key); verifyInstrumentedOnce(fs); Assert.assertTrue(DecoratorUtils.resolveUnderlyingObject(fs) instanceof LocalFileSystem); }
public DummyJobContext(Properties jobProps, Logger logger, Map<String, JobState.DatasetState> datasetStateMap) throws Exception { super(jobProps, logger, SharedResourcesBrokerFactory .createDefaultTopLevelBroker(ConfigFactory.empty(), GobblinScopeTypes.GLOBAL.defaultScopeInstance())); this.datasetStateMap = datasetStateMap; }
@Test public void testGetNonThreadSafePublisher() throws IOException { SharedResourcesBroker broker = SharedResourcesBrokerFactory.<SimpleScopeType>createDefaultTopLevelBroker(ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance()); DataPublisher publisher1 = DataPublisherFactory.get(TestNonThreadsafeDataPublisher.class.getName(), null, broker); DataPublisher publisher2 = DataPublisherFactory.get(TestNonThreadsafeDataPublisher.class.getName(), null, broker); // should get different publishers Assert.assertNotEquals(publisher1, publisher2); // Check capabilities Assert.assertTrue(publisher1.supportsCapability(DataPublisher.REUSABLE, Collections.EMPTY_MAP)); Assert.assertFalse(publisher1.supportsCapability(Capability.THREADSAFE, Collections.EMPTY_MAP)); }
public SharedResourcesBroker<GobblinScopeTypes> getDefaultInstanceBroker() { getLog().warn("Creating a default instance broker for job launcher. Objects may not be shared across all jobs in this instance."); SharedResourcesBrokerImpl<GobblinScopeTypes> globalBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(getSysConfig().getConfig(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); return globalBroker.newSubscopedBuilder(new SimpleScope<>(GobblinScopeTypes.INSTANCE, getInstanceName())).build(); }
@Test public void testCreationWithInstrumentedScheme() throws Exception { SharedResourcesBrokerImpl<SimpleScopeType> broker = SharedResourcesBrokerFactory.<SimpleScopeType>createDefaultTopLevelBroker( ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance()); FileSystemKey key = new FileSystemKey(new URI("instrumented-file:///"), new Configuration()); FileSystemFactory<SimpleScopeType> factory = new FileSystemFactory<>(); FileSystem fs = broker.getSharedResource(factory, key); verifyInstrumentedOnce(fs); Assert.assertTrue(DecoratorUtils.resolveUnderlyingObject(fs) instanceof LocalFileSystem); }
public static SharedResourcesBroker<GobblinScopeTypes> getSharedJobBroker(Properties props) { SharedResourcesBroker<GobblinScopeTypes> instanceBroker = SharedResourcesBrokerFactory .createDefaultTopLevelBroker(ConfigFactory.parseProperties(props), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBroker<GobblinScopeTypes> jobBroker = instanceBroker .newSubscopedBuilder(new JobScopeInstance("ConvertibleHiveDatasetLineageEventTest", String.valueOf(System.currentTimeMillis()))) .build(); return jobBroker; } }
@Test public void test() throws Exception { SharedResourcesBrokerImpl<SimpleScopeType> broker = SharedResourcesBrokerFactory.<SimpleScopeType>createDefaultTopLevelBroker( ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance()); FileSystemKey key = new FileSystemKey(new URI("file:///"), new Configuration()); FileSystemFactory<SimpleScopeType> factory = new FileSystemFactory<>(); FileSystem fs = broker.getSharedResource(factory, key); verifyInstrumentedOnce(fs); SharedResourcesBroker<SimpleScopeType> subBroker = broker.newSubscopedBuilder(SimpleScopeType.LOCAL.defaultScopeInstance()).build(); FileSystem subBrokerFs = FileSystemFactory.get(new URI("file:///"), new Configuration(), subBroker); Assert.assertEquals(fs, subBrokerFs); }
public SharedResourcesBroker<GobblinScopeTypes> getDefaultInstanceBroker() { SharedResourcesBrokerImpl<GobblinScopeTypes> globalBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(getSysConfig().getConfig(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); return globalBroker.newSubscopedBuilder(new SimpleScope<>(GobblinScopeTypes.INSTANCE, getInstanceName())).build(); }
public SharedResourcesBroker<GobblinScopeTypes> getDefaultInstanceBroker() { SharedResourcesBrokerImpl<GobblinScopeTypes> globalBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(getSysConfig().getConfig(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); return globalBroker.newSubscopedBuilder(new SimpleScope<>(GobblinScopeTypes.INSTANCE, getInstanceName())).build(); }
@Test public void testFailIfIntermediateScopeHasNoDefault() throws Exception { Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); // should trow error if an intermediate scope does not have a default try { topBroker.newSubscopedBuilder(new TaskScopeInstance("taskxyz")).build(); Assert.fail(); } catch (IllegalArgumentException iae) { // expected } }
/** * Create a {@link WorkUnitState} with a {@link org.apache.gobblin.broker.iface.SharedResourcesBroker} for running unit tests of * constructs. */ public static WorkUnitState createTestWorkUnitState() { return new WorkUnitState(new WorkUnit(), new State(), SharedResourcesBrokerFactory.createDefaultTopLevelBroker( ConfigFactory.empty(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()). newSubscopedBuilder(new JobScopeInstance("jobName", "testJob"))); }
@Test public void testExpiringResource() throws Exception { Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<SimpleScopeType> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, SimpleScopeType.GLOBAL.defaultScopeInstance()); InvalidatableResourceFactory factory = new InvalidatableResourceFactory(); long value = topBroker.getSharedResource(factory, new EmptyKey()); Assert.assertEquals(topBroker.getSharedResource(factory, new EmptyKey()), new Long(value)); factory.getLastResourceEntry().setValid(false); Assert.assertNotEquals(topBroker.getSharedResource(factory, new EmptyKey()), value); value = topBroker.getSharedResource(factory, new EmptyKey()); Assert.assertEquals(topBroker.getSharedResource(factory, new EmptyKey()), new Long(value)); }
@Test public void testFactory() throws Exception { SharedResourcesBroker<SimpleScopeType> broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker( ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance()); MyRequestSender requestSender = new MyRequestSender(); broker.bindSharedResourceAtScope(new RedirectAwareRestClientRequestSender.Factory<>(), new SharedRestClientKey(RestliLimiterFactory.RESTLI_SERVICE_NAME), SimpleScopeType.GLOBAL, requestSender); RestliServiceBasedLimiter limiter = broker.getSharedResource(new RestliLimiterFactory<>(), new SharedLimiterKey("my/resource")); Assert.assertNotNull(limiter.acquirePermits(10)); Assert.assertEquals(requestSender.requestList.size(), 1); broker.close(); }
@Test public void testRestliLimiterCalledByLimiterFactory() throws Exception { SharedResourcesBroker<SimpleScopeType> broker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker( ConfigFactory.empty(), SimpleScopeType.GLOBAL.defaultScopeInstance()); MyRequestSender requestSender = new MyRequestSender(); broker.bindSharedResourceAtScope(new RedirectAwareRestClientRequestSender.Factory<>(), new SharedRestClientKey(RestliLimiterFactory.RESTLI_SERVICE_NAME), SimpleScopeType.GLOBAL, requestSender); Limiter limiter = broker.getSharedResource(new SharedLimiterFactory<>(), new SharedLimiterKey("my/resource")); Assert.assertNotNull(limiter.acquirePermits(10)); Assert.assertEquals(requestSender.requestList.size(), 1); broker.close(); }