protected void initDispatcher(Configuration conf) { // create async handler dispatcher = new AsyncDispatcher("AttributeNodeLabelsManager dispatcher"); AsyncDispatcher asyncDispatcher = (AsyncDispatcher) dispatcher; asyncDispatcher.init(conf); asyncDispatcher.setDrainEventsOnStop(); }
protected void initDispatcher(Configuration conf) { // create async handler dispatcher = new AsyncDispatcher(); AsyncDispatcher asyncDispatcher = (AsyncDispatcher) dispatcher; asyncDispatcher.init(conf); asyncDispatcher.setDrainEventsOnStop(); }
protected void initDispatcher(Configuration conf) { // create async handler dispatcher = new AsyncDispatcher("NodeLabelManager dispatcher"); AsyncDispatcher asyncDispatcher = (AsyncDispatcher) dispatcher; asyncDispatcher.init(conf); asyncDispatcher.setDrainEventsOnStop(); }
protected void initDispatcher(Configuration conf) { // create async handler dispatcher = new AsyncDispatcher(); AsyncDispatcher asyncDispatcher = (AsyncDispatcher) dispatcher; asyncDispatcher.init(conf); asyncDispatcher.setDrainEventsOnStop(); }
@Override protected void serviceInit(Configuration conf) throws Exception{ // create async handler dispatcher = new AsyncDispatcher("RM StateStore dispatcher"); dispatcher.init(conf); rmStateStoreEventHandler = new ForwardingEventHandler(); dispatcher.register(RMStateStoreEventType.class, rmStateStoreEventHandler); dispatcher.setDrainEventsOnStop(); // read the base epoch value from conf baseEpoch = conf.getLong(YarnConfiguration.RM_EPOCH, YarnConfiguration.DEFAULT_RM_EPOCH); epochRange = conf.getLong(YarnConfiguration.RM_EPOCH_RANGE, YarnConfiguration.DEFAULT_RM_EPOCH_RANGE); initInternal(conf); }
@Override protected void serviceInit(Configuration conf) throws Exception{ // create async handler dispatcher = new AsyncDispatcher(); dispatcher.init(conf); dispatcher.register(RMStateStoreEventType.class, new ForwardingEventHandler()); dispatcher.setDrainEventsOnStop(); initInternal(conf); }
@Override protected void serviceInit(Configuration conf) throws Exception{ // create async handler dispatcher = new AsyncDispatcher(); dispatcher.init(conf); dispatcher.register(RMStateStoreEventType.class, new ForwardingEventHandler()); dispatcher.setDrainEventsOnStop(); initInternal(conf); }
@Test(timeout=20000) public void testRebootedDuringCommit() throws Exception { Configuration conf = new Configuration(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); conf.setInt(MRJobConfig.MR_AM_MAX_ATTEMPTS, 2); AsyncDispatcher dispatcher = new AsyncDispatcher(); dispatcher.init(conf); dispatcher.start(); CyclicBarrier syncBarrier = new CyclicBarrier(2); OutputCommitter committer = new WaitingOutputCommitter(syncBarrier, true); CommitterEventHandler commitHandler = createCommitterEventHandler(dispatcher, committer); commitHandler.init(conf); commitHandler.start(); AppContext mockContext = mock(AppContext.class); when(mockContext.isLastAMRetry()).thenReturn(true); when(mockContext.hasSuccessfullyUnregistered()).thenReturn(false); JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, mockContext); completeJobTasks(job); assertJobState(job, JobStateInternal.COMMITTING); syncBarrier.await(); job.handle(new JobEvent(job.getID(), JobEventType.JOB_AM_REBOOT)); assertJobState(job, JobStateInternal.REBOOT); // return the external state as ERROR since this is last retry. Assert.assertEquals(JobState.RUNNING, job.getState()); when(mockContext.hasSuccessfullyUnregistered()).thenReturn(true); Assert.assertEquals(JobState.ERROR, job.getState()); dispatcher.stop(); commitHandler.stop(); }
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); AsyncDispatcher dispatcher = new AsyncDispatcher(); dispatcher.init(conf); dispatcher.start();
@Test(timeout=20000) public void testCommitJobFailsJob() throws Exception { Configuration conf = new Configuration(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); AsyncDispatcher dispatcher = new AsyncDispatcher(); dispatcher.init(conf); dispatcher.start(); CyclicBarrier syncBarrier = new CyclicBarrier(2); OutputCommitter committer = new TestingOutputCommitter(syncBarrier, false); CommitterEventHandler commitHandler = createCommitterEventHandler(dispatcher, committer); commitHandler.init(conf); commitHandler.start(); JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, null); completeJobTasks(job); assertJobState(job, JobStateInternal.COMMITTING); // let the committer fail and verify the job fails syncBarrier.await(); assertJobState(job, JobStateInternal.FAILED); dispatcher.stop(); commitHandler.stop(); }
dispatcher.init(conf); dispatcher.start(); OutputCommitter committer = mock(OutputCommitter.class);
schedulerDispatcher.disableExitOnError(); rmDispatcher.register(SchedulerEventType.class, schedulerDispatcher); rmDispatcher.init(conf); rmDispatcher.start(); schedulerDispatcher.init(conf);
@Test(timeout=20000) public void testKilledDuringCommit() throws Exception { Configuration conf = new Configuration(); conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); AsyncDispatcher dispatcher = new AsyncDispatcher(); dispatcher.init(conf); dispatcher.start(); CyclicBarrier syncBarrier = new CyclicBarrier(2); OutputCommitter committer = new WaitingOutputCommitter(syncBarrier, true); CommitterEventHandler commitHandler = createCommitterEventHandler(dispatcher, committer); commitHandler.init(conf); commitHandler.start(); JobImpl job = createRunningStubbedJob(conf, dispatcher, 2, null); completeJobTasks(job); assertJobState(job, JobStateInternal.COMMITTING); syncBarrier.await(); job.handle(new JobEvent(job.getID(), JobEventType.JOB_KILL)); assertJobState(job, JobStateInternal.KILLED); dispatcher.stop(); commitHandler.stop(); }
new SchedulerEventDispatcher(sched); rmDispatcher.register(SchedulerEventType.class, schedulerDispatcher); rmDispatcher.init(conf); rmDispatcher.start(); schedulerDispatcher.init(conf);
Configuration conf = new Configuration(); AsyncDispatcher dispatcher = new AsyncDispatcher(); dispatcher.init(conf); dispatcher.start();
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); AsyncDispatcher dispatcher = new AsyncDispatcher(); dispatcher.init(conf); dispatcher.start(); OutputCommitter committer = new StubbedOutputCommitter() {
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); AsyncDispatcher dispatcher = new AsyncDispatcher(); dispatcher.init(conf); dispatcher.start(); OutputCommitter committer = new StubbedOutputCommitter() {
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); AsyncDispatcher dispatcher = new AsyncDispatcher(); dispatcher.init(conf); dispatcher.start(); CyclicBarrier syncBarrier = new CyclicBarrier(2);
conf.set(MRJobConfig.MR_AM_STAGING_DIR, stagingDir); AsyncDispatcher dispatcher = new AsyncDispatcher(); dispatcher.init(conf); dispatcher.start(); OutputCommitter committer = new StubbedOutputCommitter() {
@BeforeClass public static void setUp() throws Exception { util = new TajoTestingCluster(); util.startCatalogCluster(); conf = util.getConfiguration(); conf.set(TajoConf.ConfVars.$TEST_BROADCAST_JOIN_ENABLED.varname, "false"); catalog = util.getCatalogService(); catalog.createTablespace(DEFAULT_TABLESPACE_NAME, "hdfs://localhost:!234/warehouse"); catalog.createDatabase(DEFAULT_DATABASE_NAME, DEFAULT_TABLESPACE_NAME); TPCH tpch = new TPCH(); tpch.loadSchemas(); tpch.loadOutSchema(); for (String table : tpch.getTableNames()) { TableMeta m = CatalogUtil.newTableMeta(BuiltinStorages.TEXT, util.getConfiguration()); TableDesc d = CatalogUtil.newTableDesc( IdentifierUtil.buildFQName(DEFAULT_DATABASE_NAME, table), tpch.getSchema(table), m, CommonTestingUtil.getTestDir()); TableStats stats = new TableStats(); stats.setNumBytes(TPCH.tableVolumes.get(table)); d.setStats(stats); catalog.createTable(d); } analyzer = new SQLAnalyzer(); logicalPlanner = new LogicalPlanner(catalog, TablespaceManager.getInstance()); optimizer = new LogicalOptimizer(conf, catalog, TablespaceManager.getInstance()); dispatcher = new AsyncDispatcher(); dispatcher.init(conf); dispatcher.start(); planner = new GlobalPlanner(conf, catalog); }