public void testLatestRange() throws Exception { Configuration conf = getConf(); Date now = new Date(); conf.set("start_time", DateUtils.formatDateOozieTZ(now)); conf.set("end_time", DateUtils.formatDateOozieTZ(new Date(now.getTime() + 3 * 60 * 60 * 1000))); conf.set("initial_instance_a", DateUtils.formatDateOozieTZ(new Date(now.getTime() - 5 * 60 * 60 * 1000))); conf.set("initial_instance_b", DateUtils.formatDateOozieTZ(new Date(now.getTime() - 5 * 60 * 60 * 1000))); String inputLogic = //@formatter:off "<data-in name=\"test\" dataset=\"A\" min =\"2\" />"; //@formatter:on String jobId = _testCoordSubmit("coord-inputlogic-range-latest.xml", conf, inputLogic); createDirWithTime("input-data/a/", now, 0, 1); startCoordAction(jobId); CoordinatorActionBean actionBean = CoordActionQueryExecutor.getInstance().get( CoordActionQuery.GET_COORD_ACTION, jobId + "@1"); assertFalse(CoordinatorAction.Status.WAITING.equals(actionBean.getStatus())); XConfiguration runConf = new XConfiguration(new StringReader(actionBean.getRunConf())); String dataSets = runConf.get("inputLogicData"); assertEquals(dataSets.split(",").length, 2); }
public void testWaitFail() throws Exception { Configuration conf = getConf(); Date now = new Date(); String inputLogic = //@formatter:off "<or name=\"test\" min=\"${min}\" wait=\"${wait}\">"+ "<data-in dataset=\"${dataA}\" />"+ "<data-in dataset=\"${dataB}\" />"+ "</or>"; //@formatter:on conf.set("start_time", DateUtils.formatDateOozieTZ(now)); conf.set("min", "4"); conf.set("wait", "180"); conf.set("dataA", "A"); conf.set("dataB", "B"); conf.set("end_time", DateUtils.formatDateOozieTZ(new Date(now.getTime() + 3 * 60 * 60 * 1000))); conf.set("initial_instance_a", DateUtils.formatDateOozieTZ(new Date(now.getTime() - 5 * 60 * 60 * 1000))); conf.set("initial_instance_b", DateUtils.formatDateOozieTZ(new Date(now.getTime() - 5 * 60 * 60 * 1000))); String jobId = _testCoordSubmit("coord-inputlogic-range.xml", conf, inputLogic, getInputEventForRange()); createDirWithTime("input-data/b/", now, 0, 1, 2, 3, 4); startCoordActionForWaiting(jobId); new CoordActionInputCheckXCommand(jobId + "@1", jobId).call(); CoordinatorActionBean actionBean = CoordActionQueryExecutor.getInstance().get( CoordActionQuery.GET_COORD_ACTION, jobId + "@1"); assertTrue(CoordinatorAction.Status.WAITING.equals(actionBean.getStatus())); }
public void testLatestRange() throws Exception { Configuration conf = getConfForCombine(); Date now = new Date(); conf.set("start_time", DateUtils.formatDateOozieTZ(now)); conf.set("end_time", DateUtils.formatDateOozieTZ(new Date(now.getTime() + 10 * TIME_DAYS))); conf.set("initial_instance_a", DateUtils.formatDateOozieTZ(new Date(now.getTime() - 5 * TIME_DAYS))); conf.set("initial_instance_b", DateUtils.formatDateOozieTZ(new Date(now.getTime() - 5 * TIME_DAYS))); String inputLogic = //@formatter:off "<and name=\"test\">"+ "<data-in dataset=\"A\" />" + "<data-in dataset=\"B\" />" + "</and>"; //@formatter:on String jobId = submitCoord("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.LATEST_RANGE); List<String> inputDir = createDirWithTime("input-data/b/", now, 0, 1, 2, 3, 4, 5); inputDir.addAll(createPartitionWithTime("db_a", now, 0, 1, 2, 3, 4, 5)); startCoordAction(jobId); CoordinatorActionBean actionBean = CoordActionQueryExecutor.getInstance().get( CoordActionQuery.GET_COORD_ACTION, jobId + "@1"); assertFalse(CoordinatorAction.Status.WAITING.equals(actionBean.getStatus())); XConfiguration runConf = new XConfiguration(new StringReader(actionBean.getRunConf())); String dataSets = runConf.get("inputLogicData"); assertEquals(dataSets.split(",").length, 12); checkDataSets(dataSets, inputDir.toArray(new String[inputDir.size()])); }
public void testSingeSetWithMin() throws Exception { Configuration conf = getConf(); //@formatter:off String inputLogic = "<or name=\"test\">"+ "<data-in dataset=\"A\" min=\"3\" />"+ "</or>"; //@formatter:on conf.set("initial_instance_a", "2014-10-07T00:00Z"); conf.set("initial_instance_b", "2014-10-07T00:00Z"); String jobId = _testCoordSubmit("coord-inputlogic-range.xml", conf, inputLogic, getInputEventForRange()); String input1 = createTestCaseSubDir("input-data/a/2014/10/08/00/_SUCCESS".split("/")); String input2 = createTestCaseSubDir("input-data/a/2014/10/07/23/_SUCCESS".split("/")); // dataset with gap String input3 = createTestCaseSubDir("input-data/a/2014/10/07/19/_SUCCESS".split("/")); startCoordAction(jobId); CoordinatorActionBean actionBean = CoordActionQueryExecutor.getInstance().get( CoordActionQuery.GET_COORD_ACTION, jobId + "@1"); assertFalse(CoordinatorAction.Status.WAITING.equals(actionBean.getStatus())); XConfiguration runConf = new XConfiguration(new StringReader(actionBean.getRunConf())); String dataSets = runConf.get("inputLogicData"); assertEquals(dataSets.split(",").length, 3); checkDataSets(dataSets, input1, input2, input3); }
public void testAndWithMin() throws Exception { Configuration conf = getConf(); //@formatter:off String inputLogic = "<and name=\"test\">"+ "<data-in dataset=\"A\" min=\"2\"/>"+ "<data-in dataset=\"B\" min=\"3\"/>"+ "<data-in dataset=\"C\" min=\"0\"/>"+ "</and>"; //@formatter:on conf.set("initial_instance_a", "2014-10-07T00:00Z"); conf.set("initial_instance_b", "2014-10-07T00:00Z"); String jobId = _testCoordSubmit("coord-inputlogic-range.xml", conf, inputLogic, getInputEventForRange()); String input1 = createTestCaseSubDir("input-data/a/2014/10/08/00/_SUCCESS".split("/")); String input2 = createTestCaseSubDir("input-data/a/2014/10/07/23/_SUCCESS".split("/")); String input3 = createTestCaseSubDir("input-data/b/2014/10/07/21/_SUCCESS".split("/")); String input4 = createTestCaseSubDir("input-data/b/2014/10/07/20/_SUCCESS".split("/")); String input5 = createTestCaseSubDir("input-data/b/2014/10/07/19/_SUCCESS".split("/")); startCoordAction(jobId); CoordinatorActionBean actionBean = CoordActionQueryExecutor.getInstance().get( CoordActionQuery.GET_COORD_ACTION, jobId + "@1"); assertFalse(CoordinatorAction.Status.WAITING.equals(actionBean.getStatus())); XConfiguration runConf = new XConfiguration(new StringReader(actionBean.getRunConf())); String dataSets = runConf.get("inputLogicData"); assertEquals(dataSets.split(",").length, 5); checkDataSets(dataSets, input1, input2, input3, input4, input5, input5); }
public void testHcatHdfs() throws Exception { Configuration conf = getConfForCombine(); conf.set("initial_instance_a", "2014-10-07T00:00Z"); conf.set("initial_instance_b", "2014-10-07T00:00Z"); String inputLogic = //@formatter:off "<and name=\"test\">" + "<data-in name=\"testA\" dataset=\"A\" />" + "<data-in name=\"testB\" dataset=\"B\" />" + "</and>"; //@formatter:on String jobId = submitCoord("coord-inputlogic-combine.xml", conf, inputLogic, TEST_TYPE.CURRENT_SINGLE); String input1 = createTestCaseSubDir("input-data/b/2014/10/08/_SUCCESS".split("/")); String input2 = addPartition("db_a", "table1", "dt=20141008;country=usa"); new CoordMaterializeTransitionXCommand(jobId, 3600).call(); new CoordPushDependencyCheckXCommand(jobId + "@1").call(); new CoordActionInputCheckXCommand(jobId + "@1", jobId).call(); startCoordAction(jobId); CoordinatorActionBean actionBean = CoordActionQueryExecutor.getInstance().get( CoordActionQuery.GET_COORD_ACTION, jobId + "@1"); assertFalse(CoordinatorAction.Status.WAITING.equals(actionBean.getStatus())); XConfiguration runConf = new XConfiguration(new StringReader(actionBean.getRunConf())); String dataSets = runConf.get("inputLogicData"); assertEquals(dataSets.split(",").length, 2); checkDataSets(dataSets, input1, input2); }
public void testVerifyPrecondition() throws Exception { Date startTime = new Date(); Date endTime = new Date(startTime.getTime() + 1 * 60 * 1000); CoordinatorJobBean job = addRecordToCoordJobTable(CoordinatorJob.Status.RUNNING, startTime, endTime, false, true, 0); int actionNum = 1; for (CoordinatorAction.Status actionStatus : CoordinatorAction.Status.values()) { CoordinatorActionBean action = addRecordToCoordActionTable(job.getId(), actionNum, actionStatus, "coord-action-get.xml", 0); try { new CoordActionSkipXCommand(action, getTestUser(), "my-app-name").verifyPrecondition(); if (!(actionStatus.equals(CoordinatorAction.Status.WAITING) || actionStatus.equals(CoordinatorAction.Status.READY))) { fail(); } } catch (PreconditionException pe) { assertEquals(ErrorCode.E1100, pe.getErrorCode()); assertTrue(pe.getMessage().endsWith("[" + actionStatus + "]]")); } actionNum++; } }
public void testAnd() throws Exception { Configuration conf = getConf(); //@formatter:off String inputLogic = "<and name=\"test\">"+ "<data-in dataset=\"A\"/>"+ "<data-in dataset=\"B\"/>"+ "</and>"; //@formatter:on String jobId = _testCoordSubmit("coord-inputlogic.xml", conf, inputLogic); String input1 = createTestCaseSubDir("input-data/a/2014/10/08/00/_SUCCESS".split("/")); String input2 = createTestCaseSubDir("input-data/b/2014/10/08/00/_SUCCESS".split("/")); startCoordAction(jobId); CoordinatorActionBean actionBean = CoordActionQueryExecutor.getInstance().get( CoordActionQuery.GET_COORD_ACTION, jobId + "@1"); assertFalse(CoordinatorAction.Status.WAITING.equals(actionBean.getStatus())); XConfiguration runConf = new XConfiguration(new StringReader(actionBean.getRunConf())); String dataSets = runConf.get("inputLogicData"); assertEquals(dataSets.split(",").length, 2); checkDataSets(dataSets, input1, input2); }
@Override protected void verifyPrecondition() throws CommandException, PreconditionException { // if the action has been updated, quit this command Timestamp actionCheckTs = new Timestamp(System.currentTimeMillis() - actionCheckDelay * 1000); Timestamp cactionLmt = coordAction.getLastModifiedTimestamp(); if (cactionLmt.after(actionCheckTs)) { throw new PreconditionException(ErrorCode.E1100, "The coord action :" + actionId + " has been updated. Ignore CoordActionCheckCommand!"); } if (coordAction.getStatus().equals(CoordinatorAction.Status.SUCCEEDED) || coordAction.getStatus().equals(CoordinatorAction.Status.FAILED) || coordAction.getStatus().equals(CoordinatorAction.Status.KILLED)) { throw new PreconditionException(ErrorCode.E1100, "The coord action [" + actionId + "] must not have status " + CoordinatorAction.Status.SUCCEEDED.name() + ", " + CoordinatorAction.Status.FAILED.name() + ", or " + CoordinatorAction.Status.KILLED.name() + " but has status [" + coordAction.getStatus().name() + "]"); } } }
private void startCoordAction(final String jobId, final CoordinatorAction.Status coordActionStatus) throws CommandException, JPAExecutorException { new CoordMaterializeTransitionXCommand(jobId, 3600).call(); new CoordActionInputCheckXCommand(jobId + "@1", jobId).call(); waitFor(50 * 1000, new Predicate() { public boolean evaluate() throws Exception { CoordinatorActionBean actionBean = CoordActionQueryExecutor.getInstance().get( CoordActionQuery.GET_COORD_ACTION, jobId + "@1"); return !actionBean.getStatus().equals(CoordinatorAction.Status.WAITING); } }); CoordinatorAction actionBean = CoordActionQueryExecutor.getInstance().get(CoordActionQuery.GET_COORD_ACTION, jobId + "@1"); assertFalse(actionBean.getStatus().equals(coordActionStatus)); CoordinatorJob coordJob = CoordJobQueryExecutor.getInstance().get(CoordJobQuery.GET_COORD_JOB, jobId); new CoordActionStartXCommand(actionBean.getId(), coordJob.getUser(), coordJob.getAppName(), actionBean.getJobId()).call(); }
@Override protected void verifyPrecondition() throws CommandException, PreconditionException { // if the action has been updated, quit this command Timestamp actionCheckTs = new Timestamp(System.currentTimeMillis() - actionCheckDelay * 1000); Timestamp cactionLmt = coordAction.getLastModifiedTimestamp(); if (cactionLmt.after(actionCheckTs)) { throw new PreconditionException(ErrorCode.E1100, "The coord action :" + actionId + " has been updated. Ignore CoordActionCheckCommand!"); } if (coordAction.getStatus().equals(CoordinatorAction.Status.SUCCEEDED) || coordAction.getStatus().equals(CoordinatorAction.Status.FAILED) || coordAction.getStatus().equals(CoordinatorAction.Status.KILLED)) { throw new PreconditionException(ErrorCode.E1100, "The coord action [" + actionId + "] must not have status " + CoordinatorAction.Status.SUCCEEDED.name() + ", " + CoordinatorAction.Status.FAILED.name() + ", or " + CoordinatorAction.Status.KILLED.name() + " but has status [" + coordAction.getStatus().name() + "]"); } } }
public void testLatest() throws Exception { Configuration conf = getConf(); conf.set("initial_instance_a", "2014-10-07T00:00Z"); conf.set("initial_instance_b", "2014-10-07T00:00Z"); String inputLogic = "<data-in name=\"test\" dataset=\"A\"/>"; String jobId = _testCoordSubmit("coord-inputlogic-latest.xml", conf, inputLogic); String input1 = createTestCaseSubDir("input-data/a/2014/10/08/00/_SUCCESS".split("/")); startCoordAction(jobId); CoordinatorActionBean actionBean = CoordActionQueryExecutor.getInstance().get( CoordActionQuery.GET_COORD_ACTION, jobId + "@1"); assertFalse(CoordinatorAction.Status.WAITING.equals(actionBean.getStatus())); XConfiguration runConf = new XConfiguration(new StringReader(actionBean.getRunConf())); String dataSets = runConf.get("inputLogicData"); assertEquals(dataSets.split(",").length, 1); checkDataSets(dataSets, input1); }
private void startCoordActionForWaiting(final String jobId) throws CommandException, JPAExecutorException, JDOMException { new CoordMaterializeTransitionXCommand(jobId, 3600).call(); new CoordActionInputCheckXCommand(jobId + "@1", jobId).call(); waitFor(5 * 1000, new Predicate() { public boolean evaluate() throws Exception { CoordinatorActionBean actionBean = CoordActionQueryExecutor.getInstance().get( CoordActionQuery.GET_COORD_ACTION, jobId + "@1"); return !actionBean.getStatus().equals(CoordinatorAction.Status.WAITING); } }); CoordinatorActionBean actionBean = CoordActionQueryExecutor.getInstance().get( CoordActionQuery.GET_COORD_ACTION, jobId + "@1"); assertTrue("should be waiting", actionBean.getStatus().equals(CoordinatorAction.Status.WAITING)); }
@Override public boolean evaluate() throws Exception { try { CoordinatorActionBean action = jpaService.execute(new CoordActionGetJPAExecutor(actionId)); return stat.equals(action.getStatus()); } catch (JPAExecutorException se) { throw new Exception("Action ID " + actionId + " was not stored properly in db"); } } });
public boolean evaluate() throws Exception { CoordinatorActionBean actionBean = CoordActionQueryExecutor.getInstance().get( CoordActionQuery.GET_COORD_ACTION, jobId + "@1"); return !actionBean.getStatus().equals(CoordinatorAction.Status.READY); } });
public boolean evaluate() throws Exception { CoordinatorActionBean coordAction2 = jpaService.execute(new CoordActionGetForExternalIdJPAExecutor(jobId)); return coordAction2.getStatus().equals(CoordinatorAction.Status.SUSPENDED); } });
public boolean evaluate() throws Exception { CoordinatorActionBean actionBean = CoordActionQueryExecutor.getInstance().get( CoordActionQuery.GET_COORD_ACTION, jobId + "@1"); return !actionBean.getStatus().equals(CoordinatorAction.Status.WAITING); } });
public boolean evaluate() throws Exception { CoordinatorActionBean actionBean = CoordActionQueryExecutor.getInstance().get( CoordActionQuery.GET_COORD_ACTION, jobId + "@1"); return !actionBean.getStatus().equals(CoordinatorAction.Status.WAITING); } });
public boolean evaluate() throws Exception { CoordinatorActionBean actionBean = CoordActionQueryExecutor.getInstance().get( CoordActionQuery.GET_COORD_ACTION, jobId + "@1"); return !actionBean.getStatus().equals(CoordinatorAction.Status.WAITING); } });
public boolean evaluate() throws Exception { CoordinatorActionBean actionBean = CoordActionQueryExecutor.getInstance().get( CoordActionQuery.GET_COORD_ACTION, jobId + "@1"); return !actionBean.getStatus().equals(CoordinatorAction.Status.WAITING); } });