@Override public int hashCode() { int result = super.hashCode(); result = 31 * result + taskId.hashCode(); return result; } }
@Nullable @Override public ScopeInstance<GobblinScopeTypes> defaultScopeInstance() { return this.defaultId == null ? null : new GobblinScopeInstance(this, this.defaultId); }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } if (!super.equals(o)) { return false; } TaskScopeInstance that = (TaskScopeInstance) o; return taskId.equals(that.taskId); }
private LineageInfo getLineageInfo() { SharedResourcesBroker<GobblinScopeTypes> instanceBroker = SharedResourcesBrokerFactory .createDefaultTopLevelBroker(ConfigFactory.empty(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBroker<GobblinScopeTypes> jobBroker = instanceBroker .newSubscopedBuilder(new JobScopeInstance("LineageEventTest", String.valueOf(System.currentTimeMillis()))) .build(); SharedResourcesBroker<GobblinScopeTypes> taskBroker = jobBroker .newSubscopedBuilder(new TaskScopeInstance("LineageEventTestTask" + String.valueOf(System.currentTimeMillis()))) .build(); LineageInfo obj1 = LineageInfo.getLineageInfo(jobBroker).get(); LineageInfo obj2 = LineageInfo.getLineageInfo(taskBroker).get(); Assert.assertTrue(obj1 == obj2); return obj2; }
@Test public void testCreationOfBrokers() throws Exception { // Correct creation behavior Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, new SimpleScope<GobblinScopeTypes>(GobblinScopeTypes.GLOBAL, "myGlobalScope")); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> containerBroker = topBroker.newSubscopedBuilder(GobblinScopeTypes.CONTAINER.defaultScopeInstance()).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> taskBroker = jobBroker.newSubscopedBuilder(new TaskScopeInstance("taskabc")) .withAdditionalParentBroker(containerBroker).build(); Assert.assertEquals(taskBroker.selfScope().getType(), GobblinScopeTypes.TASK); Assert.assertEquals(((TaskScopeInstance) taskBroker.selfScope()).getTaskId(), "taskabc"); Assert.assertEquals(taskBroker.getScope(GobblinScopeTypes.CONTAINER).getType(), GobblinScopeTypes.CONTAINER); Assert.assertEquals(((GobblinScopeInstance) taskBroker.getScope(GobblinScopeTypes.CONTAINER)).getScopeId(), "container"); }
@Test public void test() { GobblinScopeInstance containerScope = new GobblinScopeInstance(GobblinScopeTypes.CONTAINER, "myContainer"); Assert.assertEquals(containerScope.getScopeId(), "myContainer"); JobScopeInstance jobScope = new JobScopeInstance("myJob", "job123"); Assert.assertEquals(jobScope.getJobId(), "job123"); Assert.assertEquals(jobScope.getJobName(), "myJob"); TaskScopeInstance taskScope = new TaskScopeInstance("myTask"); Assert.assertEquals(taskScope.getTaskId(), "myTask"); try { new GobblinScopeInstance(GobblinScopeTypes.JOB, "myJob"); Assert.fail(); } catch (IllegalArgumentException iae) { // expected because should use JobScopeInstance } try { new GobblinScopeInstance(GobblinScopeTypes.TASK, "myJob"); Assert.fail(); } catch (IllegalArgumentException iae) { // expected because should use TaskScopeInstance } }
/** * Create a {@link WorkUnitState} with a {@link org.apache.gobblin.broker.iface.SharedResourcesBroker} for running unit tests of * constructs. */ public static WorkUnitState createTestWorkUnitState() { return new WorkUnitState(new WorkUnit(), new State(), SharedResourcesBrokerFactory.createDefaultTopLevelBroker( ConfigFactory.empty(), GobblinScopeTypes.GLOBAL.defaultScopeInstance()). newSubscopedBuilder(new JobScopeInstance("jobName", "testJob"))); }
@Test public void testFailIfSubBrokerAtHigherScope() throws Exception { Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); try { jobBroker.newSubscopedBuilder(new GobblinScopeInstance(GobblinScopeTypes.INSTANCE, "instance")).build(); Assert.fail(); } catch (IllegalArgumentException iae) { // expected } }
@Test public void testFailIfIntermediateScopeHasNoDefault() throws Exception { Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); // should trow error if an intermediate scope does not have a default try { topBroker.newSubscopedBuilder(new TaskScopeInstance("taskxyz")).build(); Assert.fail(); } catch (IllegalArgumentException iae) { // expected } }
private static SharedResourcesBroker<GobblinScopeTypes> createDefaultInstanceBroker(Properties jobProps) { LOG.warn("Creating a job specific {}. Objects will only be shared at the job level.", SharedResourcesBroker.class.getSimpleName()); return SharedResourcesBrokerFactory.createDefaultTopLevelBroker(ConfigFactory.parseProperties(jobProps), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); }
private SharedResourcesBroker<GobblinScopeTypes> getJobBroker(JobState jobState, SharedResourcesBroker<GobblinScopeTypes> globalBroker) { return globalBroker.newSubscopedBuilder(new JobScopeInstance(jobState.getJobName(), jobState.getJobId())).build(); }
public GobblinScopeInstance(GobblinScopeTypes type, String scopeId) { super(type, scopeId); if (!type.getBaseInstanceClass().isAssignableFrom(this.getClass())) { throw new IllegalArgumentException(String.format("A scope with type %s must be a subclass of %s.", type, type.getBaseInstanceClass())); } }
public static Config setAutoScopeLevel(Config config, GobblinScopeTypes level) { return ConfigFactory.parseMap(ImmutableMap.of( JOINER.join(BrokerConstants.GOBBLIN_BROKER_CONFIG_PREFIX, NAME, AUTOSCOPE_AT), level.name())) .withFallback(config); }
@Override public S getAutoScope(SharedResourcesBroker<S> broker, ConfigView<S, TestResourceKey> config) { if (config.getConfig().hasPath(AUTOSCOPE_AT)) { return (S) GobblinScopeTypes.valueOf(config.getConfig().getString(AUTOSCOPE_AT)); } else { return broker.selfScope().getType(); } }
@Test public void testFailIfDifferentAncestors() throws Exception { // Correct creation behavior Config config = ConfigFactory.empty(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker2 = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> containerBroker = topBroker2.newSubscopedBuilder(GobblinScopeTypes.CONTAINER.defaultScopeInstance()).build(); try { jobBroker.newSubscopedBuilder(new TaskScopeInstance("taskxyz")).withAdditionalParentBroker(containerBroker).build(); Assert.fail(); } catch (IllegalArgumentException iae) { // expected } }
public static SharedResourcesBroker<GobblinScopeTypes> getSharedJobBroker(Properties props) { SharedResourcesBroker<GobblinScopeTypes> instanceBroker = SharedResourcesBrokerFactory .createDefaultTopLevelBroker(ConfigFactory.parseProperties(props), GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBroker<GobblinScopeTypes> jobBroker = instanceBroker .newSubscopedBuilder(new JobScopeInstance("ConvertibleHiveDatasetLineageEventTest", String.valueOf(System.currentTimeMillis()))) .build(); return jobBroker; } }
public DummyJobContext(Properties jobProps, Logger logger, Map<String, JobState.DatasetState> datasetStateMap) throws Exception { super(jobProps, logger, SharedResourcesBrokerFactory .createDefaultTopLevelBroker(ConfigFactory.empty(), GobblinScopeTypes.GLOBAL.defaultScopeInstance())); this.datasetStateMap = datasetStateMap; }
@Override public int hashCode() { int result = super.hashCode(); result = 31 * result + jobName.hashCode(); result = 31 * result + jobId.hashCode(); return result; } }
@Override public boolean equals(Object o) { if (this == o) { return true; } if (o == null || getClass() != o.getClass()) { return false; } if (!super.equals(o)) { return false; } JobScopeInstance that = (JobScopeInstance) o; if (!jobName.equals(that.jobName)) { return false; } return jobId.equals(that.jobId); }
@Test public void testAutoscoping() throws Exception { Config config = ConfigFactory.empty(); config = TestFactory.setAutoScopeLevel(config, GobblinScopeTypes.JOB); SharedResourcesBrokerImpl<GobblinScopeTypes> topBroker = SharedResourcesBrokerFactory.createDefaultTopLevelBroker(config, GobblinScopeTypes.GLOBAL.defaultScopeInstance()); SharedResourcesBrokerImpl<GobblinScopeTypes> jobBroker = topBroker.newSubscopedBuilder(new JobScopeInstance("myJob", "job123")).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> containerBroker = topBroker.newSubscopedBuilder(GobblinScopeTypes.CONTAINER.defaultScopeInstance()).build(); SharedResourcesBrokerImpl<GobblinScopeTypes> taskBroker = jobBroker.newSubscopedBuilder(new TaskScopeInstance("taskabc")) .withAdditionalParentBroker(containerBroker).build(); TestFactory.SharedResource jobScopedResource = taskBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.JOB); TestFactory.SharedResource taskScopedResource = taskBroker.getSharedResourceAtScope(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey"), GobblinScopeTypes.TASK); TestFactory.SharedResource autoscopedResource = taskBroker.getSharedResource(new TestFactory<GobblinScopeTypes>(), new TestResourceKey("myKey")); Assert.assertEquals(jobScopedResource, autoscopedResource); Assert.assertNotEquals(taskScopedResource, autoscopedResource); }