@Test public void sparkTask_updates_Metrics() throws IOException { Metrics mockMetrics = mock(Metrics.class); SparkTask sparkTask = new SparkTask(); sparkTask.updateTaskMetrics(mockMetrics); verify(mockMetrics, times(1)).incrementCounter(MetricsConstant.HIVE_SPARK_TASKS); verify(mockMetrics, never()).incrementCounter(MetricsConstant.HIVE_TEZ_TASKS); verify(mockMetrics, never()).incrementCounter(MetricsConstant.HIVE_MR_TASKS); }
@Test public void testHandleInterruptedException() throws Exception { HiveConf hiveConf = new HiveConf(); SparkTask sparkTask = new SparkTask(); sparkTask.setWork(mock(SparkWork.class)); DriverContext mockDriverContext = mock(DriverContext.class); QueryState mockQueryState = mock(QueryState.class); when(mockQueryState.getConf()).thenReturn(hiveConf); sparkTask.initialize(mockQueryState, null, mockDriverContext, null); SparkJobStatus mockSparkJobStatus = mock(SparkJobStatus.class); when(mockSparkJobStatus.getMonitorError()).thenReturn(new InterruptedException()); SparkSession mockSparkSession = mock(SparkSession.class); SparkJobRef mockSparkJobRef = mock(SparkJobRef.class); when(mockSparkJobRef.monitorJob()).thenReturn(2); when(mockSparkJobRef.getSparkJobStatus()).thenReturn(mockSparkJobStatus); when(mockSparkSession.submit(any(), any())).thenReturn(mockSparkJobRef); SessionState.start(hiveConf); SessionState.get().setSparkSession(mockSparkSession); sparkTask.execute(mockDriverContext); verify(mockSparkJobRef, atLeastOnce()).cancelJob(); when(mockSparkJobStatus.getMonitorError()).thenReturn( new HiveException(new InterruptedException())); sparkTask.execute(mockDriverContext); verify(mockSparkJobRef, atLeastOnce()).cancelJob(); }
@Test public void testSparkExceptionAndMonitorError() { SparkTask sparkTask = new SparkTask(); SparkJobStatus mockSparkJobStatus = mock(SparkJobStatus.class); when(mockSparkJobStatus.getMonitorError()).thenReturn(new RuntimeException()); when(mockSparkJobStatus.getSparkJobException()).thenReturn( new ExecutionException(new SparkException(""))); sparkTask.setSparkException(mockSparkJobStatus, 3); Assert.assertTrue(sparkTask.getException() instanceof HiveException); Assert.assertEquals(((HiveException) sparkTask.getException()).getCanonicalErrorMsg(), ErrorMsg.SPARK_JOB_RUNTIME_ERROR); }
@Test public void testSetSparkExceptionWithTimeoutError() { SparkTask sparkTask = new SparkTask(); SparkJobStatus mockSparkJobStatus = mock(SparkJobStatus.class); when(mockSparkJobStatus.getMonitorError()).thenReturn(new HiveException(ErrorMsg .SPARK_JOB_MONITOR_TIMEOUT, Long.toString(60))); sparkTask.setSparkException(mockSparkJobStatus, 3); Assert.assertTrue(sparkTask.getException() instanceof HiveException); Assert.assertEquals(((HiveException) sparkTask.getException()).getCanonicalErrorMsg(), ErrorMsg.SPARK_JOB_MONITOR_TIMEOUT); Assert.assertTrue(sparkTask.getException().getMessage().contains("60s")); }
@Test public void testSetSparkExceptionWithJobError() { SparkTask sparkTask = new SparkTask(); SparkJobStatus mockSparkJobStatus = mock(SparkJobStatus.class); ExecutionException ee = new ExecutionException("Exception thrown by job", new SparkException("Job aborted due to stage failure: Not a task or OOM error")); when(mockSparkJobStatus.getSparkJobException()).thenReturn(ee); sparkTask.setSparkException(mockSparkJobStatus, 3); Assert.assertTrue(sparkTask.getException() instanceof HiveException); Assert.assertEquals(((HiveException) sparkTask.getException()).getCanonicalErrorMsg(), ErrorMsg.SPARK_JOB_RUNTIME_ERROR); Assert.assertTrue(sparkTask.getException().getMessage().contains("Not a task or OOM error")); }
@Test public void testSetSparkExceptionWithOOMError() { SparkTask sparkTask = new SparkTask(); SparkJobStatus mockSparkJobStatus = mock(SparkJobStatus.class); ExecutionException jobError = new ExecutionException( new SparkException("Container killed by YARN for exceeding memory limits")); when(mockSparkJobStatus.getSparkJobException()).thenReturn(jobError); sparkTask.setSparkException(mockSparkJobStatus, 3); Assert.assertTrue(sparkTask.getException() instanceof HiveException); Assert.assertEquals(((HiveException) sparkTask.getException()).getCanonicalErrorMsg(), ErrorMsg.SPARK_RUNTIME_OOM); }
@Test public void removeEmptySparkTask() { SparkTask grandpa = new SparkTask(); SparkWork grandpaWork = new SparkWork("grandpa"); grandpaWork.add(new MapWork()); grandpa.setWork(grandpaWork); SparkTask parent = new SparkTask(); SparkWork parentWork = new SparkWork("parent"); parentWork.add(new MapWork()); parent.setWork(parentWork); SparkTask child1 = new SparkTask(); SparkWork childWork1 = new SparkWork("child1"); childWork1.add(new MapWork()); child1.setWork(childWork1); grandpa.addDependentTask(parent); parent.addDependentTask(child1); Assert.assertEquals(grandpa.getChildTasks().size(), 1); Assert.assertEquals(child1.getParentTasks().size(), 1); if (isEmptySparkWork(parent.getWork())) { SparkUtilities.removeEmptySparkTask(parent); } Assert.assertEquals(grandpa.getChildTasks().size(), 0); Assert.assertEquals(child1.getParentTasks().size(), 0); }
/** * This test tests that Utilities.get*Tasks do not repeat themselves in the process * of extracting tasks from a given set of root tasks when given DAGs that can have * multiple paths, such as the case with Diamond-shaped DAGs common to replication. */ @Test public void testGetTasksHaveNoRepeats() { CountingWrappingTask mrTask = new CountingWrappingTask(new ExecDriver()); CountingWrappingTask tezTask = new CountingWrappingTask(new TezTask()); CountingWrappingTask sparkTask = new CountingWrappingTask(new SparkTask()); // First check - we should not have repeats in results assertEquals("No repeated MRTasks from Utilities.getMRTasks", 1, Utilities.getMRTasks(getTestDiamondTaskGraph(mrTask)).size()); assertEquals("No repeated TezTasks from Utilities.getTezTasks", 1, Utilities.getTezTasks(getTestDiamondTaskGraph(tezTask)).size()); assertEquals("No repeated TezTasks from Utilities.getSparkTasks", 1, Utilities.getSparkTasks(getTestDiamondTaskGraph(sparkTask)).size()); // Second check - the tasks we looked for must not have been accessed more than // once as a result of the traversal (note that we actually wind up accessing // 2 times , because each visit counts twice, once to check for existence, and // once to visit. assertEquals("MRTasks should have been visited only once", 2, mrTask.getDepCallCount()); assertEquals("TezTasks should have been visited only once", 2, tezTask.getDepCallCount()); assertEquals("SparkTasks should have been visited only once", 2, sparkTask.getDepCallCount()); }