private static SharedResourcesBroker<GobblinScopeTypes> createDefaultInstanceBroker(Properties jobProps) { LOG.warn("Creating a job specific {}. Objects will only be shared at the job level.", SharedResourcesBroker.class.getSimpleName()); return SharedResourcesBrokerFactory.createDefaultTopLevelBroker(ConfigFactory.parseProperties(jobProps), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); }
public GobblinScopeInstance(GobblinScopeTypes type, String scopeId) { super(type, scopeId); if (!type.getBaseInstanceClass().isAssignableFrom(this.getClass())) { throw new IllegalArgumentException(String.format("A scope with type %s must be a subclass of %s.", type, type.getBaseInstanceClass())); } }
@Test public void testConfigurationInjection() throws Exception { String key = "myKey"; Config config = ConfigFactory.parseMap(ImmutableMap.of( JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, "key1"), "value1", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, "key2"), "value2", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.CONTAINER.name(), "key2"), "value2scope", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, key, "key2"), "value2key", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.CONTAINER.name(), key, "key2"), "value2scopekey" )); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> containerBroker = topBroker.newSubscopedBuilder(GobblinScopeTypes.CONTAINER.defaultScopeInstance()).build(); // create a shared resource TestFactory.SharedResource resource = containerBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.CONTAINER); Assert.assertEquals(resource.getConfig().getString("key1"), "value1"); Assert.assertEquals(resource.getConfig().getString("key2"), "value2scopekey"); }
public static Config setAutoScopeLevel(Config config, GobblinScopeTypes level) { return ConfigFactory.parseMap(ImmutableMap.of( JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, NAME, AUTOSCOPE_AT), level.name())) .withFallback(config); }
@Override public S getAutoScope(SharedResourcesBroker<S> broker, ConfigView<S, TestResourceKey> config) { if (config.getConfig().hasPath(AUTOSCOPE_AT)) { return (S) GobblinScopeTypes.valueOf(config.getConfig().getString(AUTOSCOPE_AT)); } else { return broker.selfScope().getType(); } }
@Test public void test() { String key = "myKey"; // Correct creation behavior Config config = ConfigFactory.parseMap(ImmutableMap.<String, String>builder() .put("key1", "value1") .put("key2", "value2") .put(JOINER.join(key, "key2"), "value2key") .put(JOINER.join(GobblinScopeTypes.JOB.name(), "key2"), "value2scope") .put(JOINER.join(GobblinScopeTypes.JOB.name(), key, "key2"), "value2scopekey") .build()); KeyedScopedConfigViewImpl<GobblinScopeTypes, TestResourceKey> configView = new KeyedScopedConfigViewImpl<>(GobblinScopeTypes.JOB, new TestResourceKey(key), TestFactory.NAME, config); Assert.assertEquals(configView.getScope(), GobblinScopeTypes.JOB); Assert.assertEquals(configView.getKey().toConfigurationKey(), key); Assert.assertEquals(configView.getKeyedConfig().getString("key2"), "value2key"); Assert.assertEquals(configView.getScopedConfig().getString("key2"), "value2scope"); Assert.assertEquals(configView.getKeyedScopedConfig().getString("key2"), "value2scopekey"); Assert.assertEquals(configView.getFactorySpecificConfig().getString("key1"), "value1"); Assert.assertEquals(configView.getFactorySpecificConfig().getString("key2"), "value2"); Assert.assertEquals(configView.getConfig().getString("key2"), "value2scopekey"); Assert.assertEquals(configView.getConfig().getString("key1"), "value1"); }
public void run() throws IOException, InterruptedException { List<WorkUnit> workUnits = getWorkUnits(); JobState jobState = getJobState(); Config jobConfig = getConfigFromJobState(jobState); _logger.debug("SingleTask.run: jobId {} workUnitFilePath {} jobStateFilePath {} jobState {} jobConfig {}", _jobId, _workUnitFilePath, _jobStateFilePath, jobState, jobConfig); try (SharedResourcesBroker<GobblinScopeTypes> globalBroker = SharedResourcesBrokerFactory .createDefaultTopLevelBroker(jobConfig, GobblinScopeTypes.GLOBAL.defaultScopeInstance())) { SharedResourcesBroker<GobblinScopeTypes> jobBroker = getJobBroker(jobState, globalBroker); _taskattempt = _taskAttemptBuilder.build(workUnits.iterator(), _jobId, jobState, jobBroker); _taskattempt.runAndOptionallyCommitTaskAttempt(GobblinMultiTaskAttempt.CommitPolicy.IMMEDIATE); } }
.put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, "key1"), "value1") .put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, "key2"), "value2") .put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.CONTAINER.name(), "key2"), "value2scope") .put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, key, "key2"), "value2key") .put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.CONTAINER.name(), key, "key2"), "value2scopekey") .put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.JOB.name(), "key2"), "value2scope") .put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.JOB.name(), key, "key2"), "value2scopekey") .build()); GobblinScopeTypes.GLOBAL.defaultScopeInstance()); .put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.CONTAINER.name(), "key2"), "value2scope_o") .put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, key, "key2"), "value2key_o") .put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.CONTAINER.name(), key, "key2"), "value2scopekey_o") .put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.JOB.name(), "key2"), "value2scope_o") .put(JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.JOB.name(), key, "key2"), "value2scopekey_o") .build());
public GobblinScopeInstance(GobblinScopeTypes type, String scopeId) { super(type, scopeId); if (!type.getBaseInstanceClass().isAssignableFrom(this.getClass())) { throw new IllegalArgumentException(String.format("A scope with type %s must be a subclass of %s.", type, type.getBaseInstanceClass())); } }
public DummyJobContext(Properties jobProps, Logger logger, Map<String, JobState.DatasetState> datasetStateMap) throws Exception { super(jobProps, logger, SharedResourcesBrokerFactory .createDefaultTopLevelBroker(ConfigFactory.empty(), GobblinScopeTypes.GLOBAL.defaultScopeInstance())); this.datasetStateMap = datasetStateMap; }
@Test public void testCorrectConfigInjection() { String key = "myKey"; Config config = ConfigFactory.parseMap(ImmutableMap.of( JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, "key1"), "value1", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, "key2"), "value2", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.CONTAINER.name(), "key2"), "value2scope", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, key, "key2"), "value2key", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.CONTAINER.name(), key, "key2"), "value2scopekey" )); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); KeyedScopedConfigViewImpl<GobblinScopeTypes, TestResourceKey> configView = topBroker.getConfigView(GobblinScopeTypes.CONTAINER, new TestResourceKey(key), TestFactory.NAME); Assert.assertEquals(configView.getScope(), GobblinScopeTypes.CONTAINER); Assert.assertEquals(configView.getKey().toConfigurationKey(), key); Assert.assertEquals(configView.getKeyedConfig().getString("key2"), "value2key"); Assert.assertEquals(configView.getScopedConfig().getString("key2"), "value2scope"); Assert.assertEquals(configView.getKeyedScopedConfig().getString("key2"), "value2scopekey"); Assert.assertEquals(configView.getFactorySpecificConfig().getString("key1"), "value1"); Assert.assertEquals(configView.getFactorySpecificConfig().getString("key2"), "value2"); Assert.assertEquals(configView.getConfig().getString("key2"), "value2scopekey"); Assert.assertEquals(configView.getConfig().getString("key1"), "value1"); configView = topBroker.getConfigView(GobblinScopeTypes.TASK, new TestResourceKey(key), TestFactory.NAME); Assert.assertEquals(configView.getConfig().getString("key2"), "value2key"); }
public SharedResourcesBroker<GobblinScopeTypes> getDefaultInstanceBroker() { getLog().warn("Creating a default instance broker for job launcher. Objects may not be shared across all jobs in this instance."); SharedResourcesBrokerImpl<GobblinScopeTypes> globalBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(getSysConfig().getConfig(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); return globalBroker.newSubscopedBuilder(new SimpleScope<>(GobblinScopeTypes.INSTANCE, getInstanceName())).build(); }
public static SharedResourcesBroker<GobblinScopeTypes> getSharedJobBroker(Properties props) { SharedResourcesBroker<GobblinScopeTypes> instanceBroker = SharedResourcesBrokerFactory .createDefaultTopLevelBroker(ConfigFactory.parseProperties(props), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBroker<GobblinScopeTypes> jobBroker = instanceBroker .newSubscopedBuilder(new JobScopeInstance("ConvertibleHiveDatasetLineageEventTest", String.valueOf(System.currentTimeMillis()))) .build(); return jobBroker; } }
public SharedResourcesBroker<GobblinScopeTypes> getDefaultInstanceBroker() { SharedResourcesBrokerImpl<GobblinScopeTypes> globalBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(getSysConfig().getConfig(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); return globalBroker.newSubscopedBuilder(new SimpleScope<>(GobblinScopeTypes.INSTANCE, getInstanceName())).build(); }
public SharedResourcesBroker<GobblinScopeTypes> getDefaultInstanceBroker() { SharedResourcesBrokerImpl<GobblinScopeTypes> globalBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(getSysConfig().getConfig(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); return globalBroker.newSubscopedBuilder(new SimpleScope<>(GobblinScopeTypes.INSTANCE, getInstanceName())).build(); }
@Test private void testCreateGobblinScopedCustomPusher() throws NotConfiguredException { Map<String, String> configAsMap = new HashMap<>(); configAsMap.put("gobblin.broker.pusher.class", TestPusher.class.getName()); configAsMap.put("gobblin.broker.pusher.id", "sharedId"); configAsMap.put("gobblin.broker.pusher.testPusher.id", "testPusherId"); configAsMap.put("gobblin.broker.pusher.testPusher.name", "testPusherName"); SharedResourcesBroker<GobblinScopeTypes> instanceBroker = SharedResourcesBrokerFactory .createDefaultTopLevelBroker(ConfigFactory.parseMap(configAsMap), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBroker<GobblinScopeTypes> jobBroker = instanceBroker.newSubscopedBuilder( new JobScopeInstance("PusherFactoryTest", String.valueOf(System.currentTimeMillis()))).build(); StringNameSharedResourceKey key = new StringNameSharedResourceKey("testPusher"); Pusher<String> pusher = jobBroker.getSharedResource(new GobblinScopePusherFactory<>(), key); Assert.assertEquals(pusher.getClass(), TestPusher.class); TestPusher testPusher = (TestPusher) pusher; Assert.assertTrue(!testPusher.isClosed); Assert.assertEquals(testPusher.id, "testPusherId"); Assert.assertEquals(testPusher.name, "testPusherName"); try { jobBroker.close(); instanceBroker.close(); } catch (IOException e) { e.printStackTrace(); } Assert.assertTrue(testPusher.isClosed); }
@Test public void testFailIfIntermediateScopeHasNoDefault() throws Exception { Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); // should trow error if an intermediate scope does not have a default try { topBroker.newSubscopedBuilder(new TaskScopeInstance("taskxyz")).build(); Assert.fail(); } catch (IllegalArgumentException iae) { // expected } }
private LineageInfo getLineageInfo() { SharedResourcesBroker<GobblinScopeTypes> instanceBroker = SharedResourcesBrokerFactory .createDefaultTopLevelBroker(ConfigFactory.empty(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBroker<GobblinScopeTypes> jobBroker = instanceBroker .newSubscopedBuilder(new JobScopeInstance("LineageEventTest", String.valueOf(System.currentTimeMillis()))) .build(); SharedResourcesBroker<GobblinScopeTypes> taskBroker = jobBroker .newSubscopedBuilder(new TaskScopeInstance("LineageEventTestTask" + String.valueOf(System.currentTimeMillis()))) .build(); LineageInfo obj1 = LineageInfo.getLineageInfo(jobBroker).get(); LineageInfo obj2 = LineageInfo.getLineageInfo(taskBroker).get(); Assert.assertTrue(obj1 == obj2); return obj2; }
@Test private void testCreateGobblinScopedDefaultPusher() throws NotConfiguredException { SharedResourcesBroker<GobblinScopeTypes> instanceBroker = SharedResourcesBrokerFactory .createDefaultTopLevelBroker(ConfigFactory.empty(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBroker<GobblinScopeTypes> jobBroker = instanceBroker.newSubscopedBuilder( new JobScopeInstance("PusherFactoryTest", String.valueOf(System.currentTimeMillis()))).build(); StringNameSharedResourceKey key = new StringNameSharedResourceKey("test"); Pusher<Object> pusher = jobBroker.getSharedResource(new GobblinScopePusherFactory<>(), key); Assert.assertEquals(pusher.getClass(), LoggingPusher.class); try { jobBroker.close(); instanceBroker.close(); } catch (IOException e) { e.printStackTrace(); } }
@Test public void testScoping() throws Exception { // Correct creation behavior Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); Assert.assertEquals(jobBroker.getScope(GobblinScopeTypes.INSTANCE).getType(), GobblinScopeTypes.INSTANCE); Assert.assertEquals(jobBroker.getScope(GobblinScopeTypes.INSTANCE).getClass(), GobblinScopeInstance.class); Assert.assertEquals(jobBroker.getScope(GobblinScopeTypes.INSTANCE), GobblinScopeTypes.INSTANCE.defaultScopeInstance()); Assert.assertEquals(jobBroker.getScope(GobblinScopeTypes.JOB).getType(), GobblinScopeTypes.JOB); Assert.assertEquals(jobBroker.getScope(GobblinScopeTypes.JOB).getClass(), JobScopeInstance.class); Assert.assertEquals(((JobScopeInstance) jobBroker.getScope(GobblinScopeTypes.JOB)).getJobId(), "job123"); try { jobBroker.getScope(GobblinScopeTypes.TASK); Assert.fail(); } catch (NoSuchScopeException nsse) { // should throw no scope exception } }