/** * Test if a shell script could run successfully with {@link ShellMain#CONF_OOZIE_SHELL_SETUP_HADOOP_CONF_DIR} enabled. * Run with {@link ShellMain#CONF_OOZIE_SHELL_SETUP_HADOOP_CONF_DIR_WRITE_LOG4J_PROPERTIES} disabled. * * @throws Exception */ public void testShellScriptHadoopConfDirWithNoL4J() throws Exception { FileSystem fs = getFileSystem(); // Create the script file with canned shell command Path script = new Path(getAppPath(), SHELL_SCRIPTNAME); Writer w = new OutputStreamWriter(fs.create(script)); w.write(SHELL_SCRIPT_LOG4J_EXISTENCE_CHECKER); w.close(); // Create sample Shell action xml String actionXml = "<shell>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<configuration>" + "<property><name>oozie.action.shell.setup.hadoop.conf.dir</name><value>true</value></property>" + "<property><name>oozie.action.shell.setup.hadoop.conf.dir.write.log4j.properties" + "</name><value>false</value></property>" + "</configuration>" + "<exec>" + SHELL_EXEC + "</exec>" + "<argument>" + SHELL_PARAM + "</argument>" + "<argument>" + SHELL_SCRIPTNAME + "</argument>" + "<file>" + script.toString() + "#" + script.getName() + "</file>" + "<capture-output/>" + "</shell>"; // Submit and verify the job's status WorkflowAction action = _testSubmit(actionXml, true, ""); String log4jExists = PropertiesUtils.stringToProperties(action.getData()).getProperty("L4J_EXISTS"); Assert.assertNull( "Expected no log4j.properties file to exist", log4jExists); }
public void testProperties() { WorkflowAction action = createNode(); Assert.assertEquals("a", action.getId()); Assert.assertEquals("b", action.getName()); Assert.assertEquals("c", action.getType()); Assert.assertEquals("d", action.getConf()); Assert.assertEquals(WorkflowAction.Status.RUNNING, action.getStatus()); Assert.assertEquals(1, action.getRetries()); Assert.assertEquals(JsonUtils.parseDateRfc822(START_TIME), action.getStartTime()); Assert.assertEquals(JsonUtils.parseDateRfc822(END_TIME), action.getEndTime()); Assert.assertEquals("e", action.getTransition()); Assert.assertEquals("ee", action.getData()); Assert.assertEquals("stats", action.getStats()); Assert.assertEquals("extChIDs", action.getExternalChildIDs()); Assert.assertEquals("f", action.getExternalId()); Assert.assertEquals("g", action.getExternalStatus()); Assert.assertEquals("h", action.getTrackerUri()); Assert.assertEquals("i", action.getConsoleUrl()); Assert.assertEquals("j", action.getErrorCode()); Assert.assertEquals("k", action.getErrorMessage()); }
public void testOutputSubmitOK() throws Exception { String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<main-class>" + LauncherMainTester.class.getName() + "</main-class>" + "<arg>out</arg>" + "<capture-output/>" + "</java>"; Context context = createContext(actionXml, null); final String runningJob = submitAction(context); waitUntilYarnAppDoneAndAssertSuccess(runningJob); ActionExecutor ae = new JavaActionExecutor(); ae.check(context, context.getAction()); assertEquals("SUCCEEDED", context.getAction().getExternalStatus()); assertNotNull(context.getAction().getData()); StringReader sr = new StringReader(context.getAction().getData()); Properties props = new Properties(); props.load(sr); assertEquals("A", props.get("a")); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus()); }
assertEquals(envValueHavingEqualSign, PropertiesUtils.stringToProperties(action.getData()) .getProperty("var2"));
public void testAdditionalJarSubmitOK() throws Exception { Path appJarPath = new Path("test-extra.jar"); File jarFile = IOUtils.createJar(new File(getTestCaseDir()), appJarPath.getName(), LauncherMainTester2.class); InputStream is = new FileInputStream(jarFile); OutputStream os = getFileSystem().create(new Path(getAppPath(), appJarPath.toString())); IOUtils.copyStream(is, os); String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<main-class>" + LauncherMainTester2.class.getName() + "</main-class>" + "<file>" + appJarPath.toString() + "</file>" + "</java>"; Context context = createContext(actionXml, null); final String runningJobId = submitAction(context); ActionExecutor ae = new JavaActionExecutor(); assertFalse(ae.isCompleted(context.getAction().getExternalStatus())); waitUntilYarnAppDoneAndAssertSuccess(runningJobId); ae.check(context, context.getAction()); assertEquals("SUCCEEDED", context.getAction().getExternalStatus()); assertNull(context.getAction().getData()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus()); }
ae.check(context, context.getAction()); assertEquals("SUCCEEDED", context.getAction().getExternalStatus()); assertNull(context.getAction().getData());
public void testSimpestSleSubmitOK() throws Exception { String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<main-class>" + LauncherMainTester.class.getName() + "</main-class>" + "</java>"; Context context = createContext(actionXml, null); submitAction(context); waitUntilYarnAppDoneAndAssertSuccess(context.getAction().getExternalId()); ActionExecutor ae = new JavaActionExecutor(); ae.check(context, context.getAction()); assertEquals("SUCCEEDED", context.getAction().getExternalStatus()); assertNull(context.getAction().getData()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus()); }
public void testSimplestSubmitWithResourceManagerOK() throws Exception { final String actionXml = "<java>" + "<resource-manager>" + getJobTrackerUri() + "</resource-manager>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<main-class>" + LauncherMainTester.class.getName() + "</main-class>" + "</java>"; final Context context = createContext(actionXml, null); submitAction(context); waitUntilYarnAppDoneAndAssertSuccess(context.getAction().getExternalId()); final ActionExecutor ae = new JavaActionExecutor(); ae.check(context, context.getAction()); assertEquals("SUCCEEDED", context.getAction().getExternalStatus()); assertNull(context.getAction().getData()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus()); }
public void testSubmitOKWithLauncherJavaOpts() throws Exception { String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<configuration>" + " <property><name>oozie.launcher.javaopts</name><value>-DtestJavaOpts=true</value></property>" + "</configuration>" + "<main-class>" + LauncherMainTester.class.getName() + "</main-class>" + "</java>"; Context context = createContext(actionXml, null); submitAction(context); waitUntilYarnAppDoneAndAssertSuccess(context.getAction().getExternalId()); ActionExecutor ae = new JavaActionExecutor(); ae.check(context, context.getAction()); assertEquals("FAILED/KILLED", context.getAction().getExternalStatus()); assertNull(context.getAction().getData()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus()); }
public void testSubmitOKWithLauncherEnvVars() throws Exception { String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<configuration>" + " <property><name>oozie.launcher.env</name><value>A=foo1" + File.pathSeparator + "B=foo2</value></property>" + "</configuration>" + "<main-class>" + LauncherMainTester.class.getName() + "</main-class>" + "</java>"; Context context = createContext(actionXml, null); submitAction(context); waitUntilYarnAppDoneAndAssertSuccess(context.getAction().getExternalId()); ActionExecutor ae = new JavaActionExecutor(); ae.check(context, context.getAction()); assertEquals("SUCCEEDED", context.getAction().getExternalStatus()); assertNull(context.getAction().getData()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus()); }
public void testSubmitOKWithLauncherJavaOptsExhaustingHeap() throws Exception { final String actionXml = "<java>" + " <job-tracker>" + getJobTrackerUri() + "</job-tracker>" + " <name-node>" + getNameNodeUri() + "</name-node>" + " <configuration>" + " <property>" + " <name>oozie.launcher.javaopts</name>" + " <value>-Xms512m -Xmx1536m -XX:-DisableExplicitGC</value>" + " </property>" + " </configuration>" + " <main-class>" + LauncherMainTester.class.getName() + "</main-class>" + " <arg>-Xmx3072m</arg>" + "</java>"; final Context context = createContext(actionXml, null); submitAction(context); waitUntilYarnAppDoneAndAssertSuccess(context.getAction().getExternalId()); ActionExecutor ae = new JavaActionExecutor(); ae.check(context, context.getAction()); assertEquals("FAILED/KILLED", context.getAction().getExternalStatus()); assertNull(context.getAction().getData()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus()); assertTrue("error message should contain: \"Java heap space\"", context.getAction().getErrorMessage().contains("Java heap space")); } }
public void testEnvVarsPropagatedFromLauncherConfig() throws Exception { String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<configuration>" + " <property><name>oozie.launcher.env</name><value>LAUNCHER_ENVIRON=foo1" + File.pathSeparator + "B=foo2</value></property>" + "</configuration>" + "<main-class>" + LauncherMainTester.class.getName() + "</main-class>" + "</java>"; Context context = createContext(actionXml, null); submitAction(context); waitUntilYarnAppDoneAndAssertSuccess(context.getAction().getExternalId()); ActionExecutor ae = new JavaActionExecutor(); ae.check(context, context.getAction()); assertEquals("FAILED/KILLED", context.getAction().getExternalStatus()); assertNull(context.getAction().getData()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus()); }
public void testSubmitOKWithVcoresAndMemory() throws Exception { String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<configuration>" + " <property><name>oozie.launcher.vcores</name><value>1</value></property>" + " <property><name>oozie.launcher.memory.mb</name><value>1024</value></property>" + "</configuration>" + "<main-class>" + LauncherMainTester.class.getName() + "</main-class>" + "</java>"; Context context = createContext(actionXml, null); submitAction(context); waitUntilYarnAppDoneAndAssertSuccess(context.getAction().getExternalId()); ActionExecutor ae = new JavaActionExecutor(); ae.check(context, context.getAction()); assertEquals("SUCCEEDED", context.getAction().getExternalStatus()); assertNull(context.getAction().getData()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus()); }
public void testExceptionSubmitException() throws Exception { String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<main-class>" + LauncherMainTester.class.getName() + "</main-class>" + "<arg>exception</arg>" + "</java>"; Context context = createContext(actionXml, null); final String runningJob = submitAction(context); waitUntilYarnAppDoneAndAssertSuccess(runningJob); //FIXME assertFalse(LauncherHelper.isMainSuccessful(runningJob)); ActionExecutor ae = new JavaActionExecutor(); ae.check(context, context.getAction()); assertTrue(ae.isCompleted(context.getAction().getExternalStatus())); assertEquals("FAILED/KILLED", context.getAction().getExternalStatus()); assertNull(context.getAction().getData()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus()); }
public void testExit0SubmitOK() throws Exception { String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<main-class>" + LauncherMainTester.class.getName() + "</main-class>" + "<arg>exit0</arg>" + "</java>"; Context context = createContext(actionXml, null); final String runningJob = submitAction(context); waitUntilYarnAppDoneAndAssertSuccess(runningJob); ActionExecutor ae = new JavaActionExecutor(); ae.check(context, context.getAction()); assertTrue(ae.isCompleted(context.getAction().getExternalStatus())); assertEquals("SUCCEEDED", context.getAction().getExternalStatus()); assertNull(context.getAction().getData()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus()); }
public void testExceptionSubmitThrowable() throws Exception { String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<main-class>" + LauncherMainTester.class.getName() + "</main-class>" + "<arg>throwable</arg>" + "</java>"; Context context = createContext(actionXml, null); final String runningJob = submitAction(context); waitUntilYarnAppDoneAndAssertSuccess(runningJob); //FIXME assertFalse(LauncherHelper.isMainSuccessful(runningJob)); ActionExecutor ae = new JavaActionExecutor(); ae.check(context, context.getAction()); assertTrue(ae.isCompleted(context.getAction().getExternalStatus())); assertEquals("FAILED/KILLED", context.getAction().getExternalStatus()); assertNull(context.getAction().getData()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus()); }
public void testExit1SubmitError() throws Exception { String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<main-class>" + LauncherMainTester.class.getName() + "</main-class>" + "<arg>exit1</arg>" + "</java>"; Context context = createContext(actionXml, null); final String runningJob = submitAction(context); waitUntilYarnAppDoneAndAssertSuccess(runningJob); //FIXME assertFalse(LauncherHelper.isMainSuccessful(runningJob)); ActionExecutor ae = new JavaActionExecutor(); ae.check(context, context.getAction()); assertTrue(ae.isCompleted(context.getAction().getExternalStatus())); assertEquals("FAILED/KILLED", context.getAction().getExternalStatus()); assertEquals("1", context.getAction().getErrorCode()); assertNull(context.getAction().getData()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.ERROR, context.getAction().getStatus()); }
private void testEmptyArgs(boolean nullArgsAllowed, String expectedExternalStatus, WorkflowAction.Status expectedStatus) throws Exception { ConfigurationService.setBoolean(LauncherAMUtils.CONF_OOZIE_NULL_ARGS_ALLOWED, nullArgsAllowed); String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<main-class>" + LauncherMainTester.class.getName() + "</main-class>" + "<arg></arg>" + "</java>"; Context context = createContext(actionXml, null); submitAction(context); waitUntilYarnAppDoneAndAssertSuccess(context.getAction().getExternalId()); ActionExecutor ae = new JavaActionExecutor(); ae.check(context, context.getAction()); assertTrue(ae.isCompleted(context.getAction().getExternalStatus())); assertEquals(expectedExternalStatus, context.getAction().getExternalStatus()); assertNull(context.getAction().getData()); ae.end(context, context.getAction()); assertEquals(expectedStatus, context.getAction().getStatus()); }
public void testRecovery() throws Exception { final String actionXml = "<java>" + "<job-tracker>" + getJobTrackerUri() + "</job-tracker>" + "<name-node>" + getNameNodeUri() + "</name-node>" + "<main-class>" + LauncherMainTester.class.getName() + "</main-class>" + "</java>"; final Context context = createContext(actionXml, null); String launcherId = submitAction(context); waitFor(60 * 1000, new Predicate() { @Override public boolean evaluate() throws Exception { JavaActionExecutor ae = new JavaActionExecutor(); Configuration conf = ae.createBaseHadoopConf(context, XmlUtils.parseXml(actionXml)); return LauncherHelper.getRecoveryId(conf, context.getActionDir(), context.getRecoveryId()) != null; } }); final String runningJob2 = submitAction(context); assertEquals(launcherId, runningJob2); assertEquals(launcherId, context.getAction().getExternalId()); waitUntilYarnAppDoneAndAssertSuccess(launcherId); ActionExecutor ae = new JavaActionExecutor(); ae.check(context, context.getAction()); assertEquals("SUCCEEDED", context.getAction().getExternalStatus()); assertNull(context.getAction().getData()); ae.end(context, context.getAction()); assertEquals(WorkflowAction.Status.OK, context.getAction().getStatus()); }