private static SharedResourcesBroker<GobblinScopeTypes> createDefaultInstanceBroker(Properties jobProps) { LOG.warn("Creating a job specific {}. Objects will only be shared at the job level.", SharedResourcesBroker.class.getSimpleName()); return SharedResourcesBrokerFactory.createDefaultTopLevelBroker(ConfigFactory.parseProperties(jobProps), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); }
public void run() throws IOException, InterruptedException { List<WorkUnit> workUnits = getWorkUnits(); JobState jobState = getJobState(); Config jobConfig = getConfigFromJobState(jobState); _logger.debug("SingleTask.run: jobId {} workUnitFilePath {} jobStateFilePath {} jobState {} jobConfig {}", _jobId, _workUnitFilePath, _jobStateFilePath, jobState, jobConfig); try (SharedResourcesBroker<GobblinScopeTypes> globalBroker = SharedResourcesBrokerFactory .createDefaultTopLevelBroker(jobConfig, GobblinScopeTypes.GLOBAL.defaultScopeInstance())) { SharedResourcesBroker<GobblinScopeTypes> jobBroker = getJobBroker(jobState, globalBroker); _taskattempt = _taskAttemptBuilder.build(workUnits.iterator(), _jobId, jobState, jobBroker); _taskattempt.runAndOptionallyCommitTaskAttempt(GobblinMultiTaskAttempt.CommitPolicy.IMMEDIATE); } }
public DummyJobContext(Properties jobProps, Logger logger, Map<String, JobState.DatasetState> datasetStateMap) throws Exception { super(jobProps, logger, SharedResourcesBrokerFactory .createDefaultTopLevelBroker(ConfigFactory.empty(), GobblinScopeTypes.GLOBAL.defaultScopeInstance())); this.datasetStateMap = datasetStateMap; }
public SharedResourcesBroker<GobblinScopeTypes> getDefaultInstanceBroker() { getLog().warn("Creating a default instance broker for job launcher. Objects may not be shared across all jobs in this instance."); SharedResourcesBrokerImpl<GobblinScopeTypes> globalBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(getSysConfig().getConfig(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); return globalBroker.newSubscopedBuilder(new SimpleScope<>(GobblinScopeTypes.INSTANCE, getInstanceName())).build(); }
public static SharedResourcesBroker<GobblinScopeTypes> getSharedJobBroker(Properties props) { SharedResourcesBroker<GobblinScopeTypes> instanceBroker = SharedResourcesBrokerFactory .createDefaultTopLevelBroker(ConfigFactory.parseProperties(props), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBroker<GobblinScopeTypes> jobBroker = instanceBroker .newSubscopedBuilder(new JobScopeInstance("ConvertibleHiveDatasetLineageEventTest", String.valueOf(System.currentTimeMillis()))) .build(); return jobBroker; } }
@Test public void testConfigurationInjection() throws Exception { String key = "myKey"; Config config = ConfigFactory.parseMap(ImmutableMap.of( JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, "key1"), "value1", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, "key2"), "value2", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.CONTAINER.name(), "key2"), "value2scope", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, key, "key2"), "value2key", JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, TestFactory.NAME, GobblinScopeTypes.CONTAINER.name(), key, "key2"), "value2scopekey" )); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> containerBroker = topBroker.newSubscopedBuilder(GobblinScopeTypes.CONTAINER.defaultScopeInstance()).build(); // create a shared resource TestFactory.SharedResource resource = containerBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.CONTAINER); Assert.assertEquals(resource.getConfig().getString("key1"), "value1"); Assert.assertEquals(resource.getConfig().getString("key2"), "value2scopekey"); }
public SharedResourcesBroker<GobblinScopeTypes> getDefaultInstanceBroker() { SharedResourcesBrokerImpl<GobblinScopeTypes> globalBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(getSysConfig().getConfig(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); return globalBroker.newSubscopedBuilder(new SimpleScope<>(GobblinScopeTypes.INSTANCE, getInstanceName())).build(); }
public SharedResourcesBroker<GobblinScopeTypes> getDefaultInstanceBroker() { SharedResourcesBrokerImpl<GobblinScopeTypes> globalBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(getSysConfig().getConfig(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); return globalBroker.newSubscopedBuilder(new SimpleScope<>(GobblinScopeTypes.INSTANCE, getInstanceName())).build(); }
@Test public void testFailIfIntermediateScopeHasNoDefault() throws Exception { Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); // should trow error if an intermediate scope does not have a default try { topBroker.newSubscopedBuilder(new TaskScopeInstance("taskxyz")).build(); Assert.fail(); } catch (IllegalArgumentException iae) { // expected } }
private LineageInfo getLineageInfo() { SharedResourcesBroker<GobblinScopeTypes> instanceBroker = SharedResourcesBrokerFactory .createDefaultTopLevelBroker(ConfigFactory.empty(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBroker<GobblinScopeTypes> jobBroker = instanceBroker .newSubscopedBuilder(new JobScopeInstance("LineageEventTest", String.valueOf(System.currentTimeMillis()))) .build(); SharedResourcesBroker<GobblinScopeTypes> taskBroker = jobBroker .newSubscopedBuilder(new TaskScopeInstance("LineageEventTestTask" + String.valueOf(System.currentTimeMillis()))) .build(); LineageInfo obj1 = LineageInfo.getLineageInfo(jobBroker).get(); LineageInfo obj2 = LineageInfo.getLineageInfo(taskBroker).get(); Assert.assertTrue(obj1 == obj2); return obj2; }
@Test private void testCreateGobblinScopedDefaultPusher() throws NotConfiguredException { SharedResourcesBroker<GobblinScopeTypes> instanceBroker = SharedResourcesBrokerFactory .createDefaultTopLevelBroker(ConfigFactory.empty(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBroker<GobblinScopeTypes> jobBroker = instanceBroker.newSubscopedBuilder( new JobScopeInstance("PusherFactoryTest", String.valueOf(System.currentTimeMillis()))).build(); StringNameSharedResourceKey key = new StringNameSharedResourceKey("test"); Pusher<Object> pusher = jobBroker.getSharedResource(new GobblinScopePusherFactory<>(), key); Assert.assertEquals(pusher.getClass(), LoggingPusher.class); try { jobBroker.close(); instanceBroker.close(); } catch (IOException e) { e.printStackTrace(); } }
@Test public void testScoping() throws Exception { // Correct creation behavior Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); Assert.assertEquals(jobBroker.getScope(GobblinScopeTypes.INSTANCE).getType(), GobblinScopeTypes.INSTANCE); Assert.assertEquals(jobBroker.getScope(GobblinScopeTypes.INSTANCE).getClass(), GobblinScopeInstance.class); Assert.assertEquals(jobBroker.getScope(GobblinScopeTypes.INSTANCE), GobblinScopeTypes.INSTANCE.defaultScopeInstance()); Assert.assertEquals(jobBroker.getScope(GobblinScopeTypes.JOB).getType(), GobblinScopeTypes.JOB); Assert.assertEquals(jobBroker.getScope(GobblinScopeTypes.JOB).getClass(), JobScopeInstance.class); Assert.assertEquals(((JobScopeInstance) jobBroker.getScope(GobblinScopeTypes.JOB)).getJobId(), "job123"); try { jobBroker.getScope(GobblinScopeTypes.TASK); Assert.fail(); } catch (NoSuchScopeException nsse) { // should throw no scope exception } }
@Test public void testFailIfDifferentAncestors() throws Exception { // Correct creation behavior Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker2 = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> containerBroker = topBroker2.newSubscopedBuilder(GobblinScopeTypes.CONTAINER.defaultScopeInstance()).build(); try { jobBroker.newSubscopedBuilder(new TaskScopeInstance("taskxyz")).withAdditionalParentBroker(containerBroker).build(); Assert.fail(); } catch (IllegalArgumentException iae) { // expected } }
/** * Create a {@link WorkUnitState} with a {@link org.apache.gobblin.broker.iface.SharedResourcesBroker} for running unit tests of * constructs. */ public static WorkUnitState createTestWorkUnitState() { return new WorkUnitState(new WorkUnit(), new State(), SharedResourcesBrokerFactory.createDefaultTopLevelBroker( ConfigFactory.empty(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()). newSubscopedBuilder(new JobScopeInstance("jobName", "testJob"))); }
@Test public void testFailIfSubBrokerAtHigherScope() throws Exception { Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); try { jobBroker.newSubscopedBuilder(new GobblinScopeInstance(GobblinScopeTypes.INSTANCE, "instance")).build(); Assert.fail(); } catch (IllegalArgumentException iae) { // expected } }
@Test public void testRedirect() throws Exception { Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); // create a shared resource TestFactory.SharedResource resource = jobBroker.getSharedResourceAtScope(new TestFactoryWithRedirect<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.JOB); Assert.assertEquals(resource.getKey(), "myKey"); Assert.assertEquals(topBroker.getSharedResourceAtScope(new TestFactoryWithRedirect<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.GLOBAL), resource); Assert.assertEquals(topBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.GLOBAL), resource); }
@Test public void testCreationOfBrokers() throws Exception { // Correct creation behavior Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, new SimpleScope<GobblinScopeTypes>(GobblinScopeTypes.GLOBAL, "myGlobalScope")); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> containerBroker = topBroker.newSubscopedBuilder(GobblinScopeTypes.CONTAINER.defaultScopeInstance()).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> taskBroker = jobBroker.newSubscopedBuilder(new TaskScopeInstance("taskabc")) .withAdditionalParentBroker(containerBroker).build(); Assert.assertEquals(taskBroker.selfScope().getType(), GobblinScopeTypes.TASK); Assert.assertEquals(((TaskScopeInstance) taskBroker.selfScope()).getTaskId(), "taskabc"); Assert.assertEquals(taskBroker.getScope(GobblinScopeTypes.CONTAINER).getType(), GobblinScopeTypes.CONTAINER); Assert.assertEquals(((GobblinScopeInstance) taskBroker.getScope(GobblinScopeTypes.CONTAINER)).getScopeId(), "container"); }
@Test public void testAutoscopedResourcesOnlyClosedInCorrectScope() throws Exception { Config config = ConfigFactory.empty(); config = TestFactory.setAutoScopeLevel(config, GobblinScopeTypes.JOB); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> containerBroker = topBroker.newSubscopedBuilder(GobblinScopeTypes.CONTAINER.defaultScopeInstance()).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> taskBroker = jobBroker.newSubscopedBuilder(new TaskScopeInstance("taskabc")) .withAdditionalParentBroker(containerBroker).build(); TestFactory.SharedResource autoscopedResource = taskBroker.getSharedResource(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey")); // since object autoscopes at job level, it should not be closed if we close the task broker taskBroker.close(); Assert.assertFalse(autoscopedResource.isClosed()); // however, when closing job broker, resource should be closed jobBroker.close(); Assert.assertTrue(autoscopedResource.isClosed()); }
@Test public void testScopedView() throws Exception { Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> instanceView = jobBroker.getScopedView(GobblinScopeTypes.INSTANCE); Assert.assertEquals(instanceView.selfScope().getType(), GobblinScopeTypes.INSTANCE); TestFactory.SharedResource resource = instanceView.getSharedResource(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey")); TestFactory.SharedResource resource2 = jobBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.INSTANCE); Assert.assertEquals(resource, resource2); try { instanceView.newSubscopedBuilder(new JobScopeInstance("otherJob", "job234")); Assert.fail(); } catch (UnsupportedOperationException exc) { // Expected } }
@Test public void testAutoscoping() throws Exception { Config config = ConfigFactory.empty(); config = TestFactory.setAutoScopeLevel(config, GobblinScopeTypes.JOB); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> containerBroker = topBroker.newSubscopedBuilder(GobblinScopeTypes.CONTAINER.defaultScopeInstance()).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> taskBroker = jobBroker.newSubscopedBuilder(new TaskScopeInstance("taskabc")) .withAdditionalParentBroker(containerBroker).build(); TestFactory.SharedResource jobScopedResource = taskBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.JOB); TestFactory.SharedResource taskScopedResource = taskBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.TASK); TestFactory.SharedResource autoscopedResource = taskBroker.getSharedResource(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey")); Assert.assertEquals(jobScopedResource, autoscopedResource); Assert.assertNotEquals(taskScopedResource, autoscopedResource); }