@Override public void closeSession(SparkSession sparkSession) throws HiveException { if (sparkSession == null) { return; } if (LOG.isDebugEnabled()) { LOG.debug(String.format("Closing Spark session (%s).", sparkSession.getSessionId())); } sparkSession.close(); createdSessions.remove(sparkSession); }
/** * If the <i>existingSession</i> can be reused return it. * Otherwise * - close it and remove it from the list. * - create a new session and add it to the list. */ @Override public SparkSession getSession(SparkSession existingSession, HiveConf conf, boolean doOpen) throws HiveException { setup(conf); if (existingSession != null) { // Open the session if it is closed. if (!existingSession.isOpen() && doOpen) { existingSession.open(conf); } return existingSession; } SparkSession sparkSession = new SparkSessionImpl(); if (doOpen) { sparkSession.open(conf); } if (LOG.isDebugEnabled()) { LOG.debug(String.format("New session (%s) is created.", sparkSession.getSessionId())); } createdSessions.add(sparkSession); return sparkSession; }
/** * Uploads a local file to HDFS * This method is not thread safe * * @param source * @param conf * @return * @throws IOException */ public static URI uploadToHDFS(URI source, HiveConf conf) throws IOException { Path localFile = new Path(source.getPath()); Path remoteFile = new Path(SessionState.get().getSparkSession().getHDFSSessionDir(), getFileName(source)); FileSystem fileSystem = FileSystem.get(remoteFile.toUri(), conf); // Overwrite if the remote file already exists. Whether the file can be added // on executor is up to spark, i.e. spark.files.overwrite fileSystem.copyFromLocalFile(false, true, localFile, remoteFile); Path fullPath = fileSystem.getFileStatus(remoteFile).getPath(); return fullPath.toUri(); }
@Override public void run() { try { SessionState.setCurrentSessionState(ss); Random random = new Random(Thread.currentThread().getId()); String threadName = Thread.currentThread().getName(); System.out.println(threadName + " started."); HiveConf conf = getHiveConf(); SparkSession prevSession = null; SparkSession currentSession = null; for(int i = 0; i < 5; i++) { currentSession = sessionManagerHS2.getSession(prevSession, conf, true); assertTrue(prevSession == null || prevSession == currentSession); assertTrue(currentSession.isOpen()); System.out.println(String.format("%s got session (%d): %s", threadName, i, currentSession.getSessionId())); Thread.sleep((random.nextInt(3)+1) * 1000); sessionManagerHS2.returnSession(currentSession); prevSession = currentSession; } sessionManagerHS2.closeSession(currentSession); System.out.println(threadName + " ended."); } catch (Throwable e) { anyFailedSessionThread = true; String msg = String.format("Error executing '%s'", Thread.currentThread().getName()); LOG.error(msg, e); fail(msg + " " + StringUtils.stringifyException(e)); } } }
@Test public void testHandleInterruptedException() throws Exception { HiveConf hiveConf = new HiveConf(); SparkTask sparkTask = new SparkTask(); sparkTask.setWork(mock(SparkWork.class)); DriverContext mockDriverContext = mock(DriverContext.class); QueryState mockQueryState = mock(QueryState.class); when(mockQueryState.getConf()).thenReturn(hiveConf); sparkTask.initialize(mockQueryState, null, mockDriverContext, null); SparkJobStatus mockSparkJobStatus = mock(SparkJobStatus.class); when(mockSparkJobStatus.getMonitorError()).thenReturn(new InterruptedException()); SparkSession mockSparkSession = mock(SparkSession.class); SparkJobRef mockSparkJobRef = mock(SparkJobRef.class); when(mockSparkJobRef.monitorJob()).thenReturn(2); when(mockSparkJobRef.getSparkJobStatus()).thenReturn(mockSparkJobStatus); when(mockSparkSession.submit(any(), any())).thenReturn(mockSparkJobRef); SessionState.start(hiveConf); SessionState.get().setSparkSession(mockSparkSession); sparkTask.execute(mockDriverContext); verify(mockSparkJobRef, atLeastOnce()).cancelJob(); when(mockSparkJobStatus.getMonitorError()).thenReturn( new HiveException(new InterruptedException())); sparkTask.execute(mockDriverContext); verify(mockSparkJobRef, atLeastOnce()).cancelJob(); }
@Override public void shutdown() { LOG.info("Closing the session manager."); synchronized (createdSessions) { Iterator<SparkSession> it = createdSessions.iterator(); while (it.hasNext()) { SparkSession session = it.next(); session.close(); } createdSessions.clear(); } inited = false; SparkClientFactory.stop(); } }
sparkSession = SparkUtilities.getSparkSession( context.getConf(), sparkSessionManager); sparkMemoryAndCores = sparkSession.getMemoryAndCores(); } catch (HiveException e) { throw new SemanticException("Failed to get a Hive on Spark session", e);
/** Tests CLI scenario where we get a single session and use it multiple times. */ @Test public void testSingleSessionMultipleUse() throws Exception { HiveConf conf = getHiveConf(); SparkSessionManager sessionManager = SparkSessionManagerImpl.getInstance(); SparkSession sparkSession1 = sessionManager.getSession(null, conf, true); assertTrue(sparkSession1.isOpen()); SparkSession sparkSession2 = sessionManager.getSession(sparkSession1, conf, true); assertTrue(sparkSession1 == sparkSession2); // Same session object is expected. assertTrue(sparkSession2.isOpen()); sessionManager.shutdown(); sessionManager.closeSession(sparkSession1); }
@Test public void testGetSessionId() throws HiveException { SessionState ss = SessionState.start(SESSION_HIVE_CONF); SparkSessionManager ssm = SparkSessionManagerImpl.getInstance(); ss.setSparkSession(ssm.getSession(null, SESSION_HIVE_CONF, true)); assertEquals(ss.getSessionId() + "_0", ss.getSparkSession().getSessionId()); ss.setSparkSession(ssm.getSession(null, SESSION_HIVE_CONF, true)); assertEquals(ss.getSessionId() + "_1", ss.getSparkSession().getSessionId()); ss = SessionState.start(SESSION_HIVE_CONF); ss.setSparkSession(ssm.getSession(null, SESSION_HIVE_CONF, true)); assertEquals(ss.getSessionId() + "_0", ss.getSparkSession().getSessionId()); }
jobRef = sparkSession.submit(driverContext, sparkWork); perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_SUBMIT_JOB);
@Override public void shutdown() { LOG.info("Closing the session manager."); if (createdSessions != null) { synchronized (createdSessions) { Iterator<SparkSession> it = createdSessions.iterator(); while (it.hasNext()) { SparkSession session = it.next(); session.close(); } createdSessions.clear(); } } inited.set(false); SparkClientFactory.stop(); } }
sparkSession = SparkUtilities.getSparkSession( context.getConf(), sparkSessionManager); sparkMemoryAndCores = sparkSession.getMemoryAndCores(); } catch (HiveException e) { throw new SemanticException("Failed to get a spark session: " + e);
/** * If the <i>existingSession</i> can be reused return it. * Otherwise * - close it and remove it from the list. * - create a new session and add it to the list. */ @Override public SparkSession getSession(SparkSession existingSession, HiveConf conf, boolean doOpen) throws HiveException { setup(conf); if (existingSession != null) { // Open the session if it is closed. if (!existingSession.isOpen() && doOpen) { existingSession.open(conf); createdSessions.add(existingSession); } return existingSession; } SparkSession sparkSession = new SparkSessionImpl(SessionState.get().getNewSparkSessionId()); if (doOpen) { sparkSession.open(conf); } if (LOG.isDebugEnabled()) { LOG.debug(String.format("New session (%s) is created.", sparkSession.getSessionId())); } createdSessions.add(sparkSession); return sparkSession; }
@Override public void closeSession(SparkSession sparkSession) throws HiveException { if (sparkSession == null) { return; } if (LOG.isDebugEnabled()) { LOG.debug(String.format("Closing session (%s).", sparkSession.getSessionId())); } sparkSession.close(); createdSessions.remove(sparkSession); }
SparkJobRef jobRef = sparkSession.submit(driverContext, sparkWork); perfLogger.PerfLogEnd(CLASS_NAME, PerfLogger.SPARK_SUBMIT_JOB);
sparkSession = SparkUtilities.getSparkSession( context.getConf(), sparkSessionManager); sparkMemoryAndCores = sparkSession.getMemoryAndCores(); } catch (Exception e) { LOG.warn("Failed to get spark memory/core info", e);
/** * Uploads a local file to HDFS * * @param source * @param conf * @return * @throws IOException */ public static URI uploadToHDFS(URI source, HiveConf conf) throws IOException { Path localFile = new Path(source.getPath()); Path remoteFile = new Path(SessionState.get().getSparkSession().getHDFSSessionDir(), getFileName(source)); FileSystem fileSystem = FileSystem.get(conf); // Overwrite if the remote file already exists. Whether the file can be added // on executor is up to spark, i.e. spark.files.overwrite fileSystem.copyFromLocalFile(false, true, localFile, remoteFile); Path fullPath = fileSystem.getFileStatus(remoteFile).getPath(); return fullPath.toUri(); }
/** * If the <i>existingSession</i> can be reused return it. * Otherwise * - close it and remove it from the list. * - create a new session and add it to the list. */ @Override public SparkSession getSession(SparkSession existingSession, HiveConf conf, boolean doOpen) throws HiveException { setup(conf); if (existingSession != null) { // Open the session if it is closed. if (!existingSession.isOpen() && doOpen) { existingSession.open(conf); } return existingSession; } SparkSession sparkSession = new SparkSessionImpl(); createdSessions.add(sparkSession); if (doOpen) { sparkSession.open(conf); } if (LOG.isDebugEnabled()) { LOG.debug(String.format("New session (%s) is created.", sparkSession.getSessionId())); } return sparkSession; }
@Override public void closeSession(SparkSession sparkSession) throws HiveException { if (sparkSession == null) { return; } if (LOG.isDebugEnabled()) { LOG.debug(String.format("Closing session (%s).", sparkSession.getSessionId())); } sparkSession.close(); createdSessions.remove(sparkSession); }