private void tearDownSessionIO() { ServiceUtils.cleanup(LOG, parentSession.getSessionState().out, parentSession.getSessionState().err); }
protected HiveCommandOperation(HiveSession parentSession, String statement, CommandProcessor commandProcessor, Map<String, String> confOverlay) { super(parentSession, statement, confOverlay, false); this.commandProcessor = commandProcessor; setupSessionIO(parentSession.getSessionState()); }
/** * Execute statement on the server. This is a blocking call. */ @Override public OperationHandle executeStatement(SessionHandle sessionHandle, String statement, Map<String, String> confOverlay) throws HiveSQLException { HiveSession session = sessionManager.getSession(sessionHandle); // need to reset the monitor, as operation handle is not available down stream, Ideally the // monitor should be associated with the operation handle. session.getSessionState().updateProgressMonitor(null); OperationHandle opHandle = session.executeStatement(statement, confOverlay); LOG.debug(sessionHandle + ": executeStatement()"); return opHandle; }
/** * Execute statement asynchronously on the server with a timeout. This is a non-blocking call */ @Override public OperationHandle executeStatementAsync(SessionHandle sessionHandle, String statement, Map<String, String> confOverlay, long queryTimeout) throws HiveSQLException { HiveSession session = sessionManager.getSession(sessionHandle); // need to reset the monitor, as operation handle is not available down stream, Ideally the // monitor should be associated with the operation handle. session.getSessionState().updateProgressMonitor(null); OperationHandle opHandle = session.executeStatementAsync(statement, confOverlay, queryTimeout); LOG.debug(sessionHandle + ": executeStatementAsync()"); return opHandle; }
/** * Execute statement on the server with a timeout. This is a blocking call. */ @Override public OperationHandle executeStatement(SessionHandle sessionHandle, String statement, Map<String, String> confOverlay, long queryTimeout) throws HiveSQLException { HiveSession session = sessionManager.getSession(sessionHandle); // need to reset the monitor, as operation handle is not available down stream, Ideally the // monitor should be associated with the operation handle. session.getSessionState().updateProgressMonitor(null); OperationHandle opHandle = session.executeStatement(statement, confOverlay, queryTimeout); LOG.debug(sessionHandle + ": executeStatement()"); return opHandle; }
/** * Execute statement asynchronously on the server. This is a non-blocking call */ @Override public OperationHandle executeStatementAsync(SessionHandle sessionHandle, String statement, Map<String, String> confOverlay) throws HiveSQLException { HiveSession session = sessionManager.getSession(sessionHandle); // need to reset the monitor, as operation handle is not available down stream, Ideally the // monitor should be associated with the operation handle. session.getSessionState().updateProgressMonitor(null); OperationHandle opHandle = session.executeStatementAsync(statement, confOverlay); LOG.debug(sessionHandle + ": executeStatementAsync()"); return opHandle; }
SessionState sessionState = getParentSession().getSessionState(); File tmp = sessionState.getTmpOutputFile(); try {
private void setupSessionIO(SessionState sessionState) { try { LOG.info("Putting temp output to file " + sessionState.getTmpOutputFile().toString() + " and error output to file " + sessionState.getTmpErrOutputFile().toString()); sessionState.in = null; // hive server's session input stream is not used // open a per-session file in auto-flush mode for writing temp results and tmp error output sessionState.out = new SessionStream( new FileOutputStream(sessionState.getTmpOutputFile()), true, StandardCharsets.UTF_8.name()); sessionState.err = new SessionStream( new FileOutputStream(sessionState.getTmpErrOutputFile()), true, StandardCharsets.UTF_8.name()); } catch (IOException e) { LOG.error("Error in creating temp output file ", e); // Close file streams to avoid resource leaking ServiceUtils.cleanup(LOG, parentSession.getSessionState().out, parentSession.getSessionState().err); try { sessionState.in = null; sessionState.out = new SessionStream(System.out, true, StandardCharsets.UTF_8.name()); sessionState.err = new SessionStream(System.err, true, StandardCharsets.UTF_8.name()); } catch (UnsupportedEncodingException ee) { LOG.error("Error creating PrintStream", e); ee.printStackTrace(); sessionState.out = null; sessionState.err = null; } } }
private void cleanTmpFile() { resetResultReader(); SessionState sessionState = getParentSession().getSessionState(); sessionState.deleteTmpOutputFile(); sessionState.deleteTmpErrOutputFile(); }
/** * Invoked after runInternal(), even if an exception is thrown in runInternal(). * Clean up resources, which was set up in beforeRun(). */ protected void afterRun() { LogUtils.unregisterLoggingContext(); // Reset back to session context after the query is done ShimLoader.getHadoopShims().setHadoopSessionContext(parentSession.getSessionState().getSessionId()); }
private double getProgressedPercentage(OperationHandle opHandle) throws HiveSQLException { checkArgument(OperationType.EXECUTE_STATEMENT.equals(opHandle.getOperationType())); Operation operation = cliService.getSessionManager().getOperationManager().getOperation(opHandle); SessionState state = operation.getParentSession().getSessionState(); ProgressMonitor monitor = state.getProgressMonitor(); return monitor == null ? 0.0 : monitor.progressedPercentage(); }
private JobProgressUpdate progressUpdateLog(boolean isProgressLogRequested, Operation operation, HiveConf conf) { if (!isProgressLogRequested || !ServiceUtils.canProvideProgressLog(conf) || !OperationType.EXECUTE_STATEMENT.equals(operation.getType())) { return new JobProgressUpdate(ProgressMonitor.NULL); } SessionState sessionState = operation.getParentSession().getSessionState(); long startTime = System.nanoTime(); int timeOutMs = 8; boolean terminated = operation.isDone(); try { while ((sessionState.getProgressMonitor() == null) && !terminated) { long remainingMs = (PROGRESS_MAX_WAIT_NS - (System.nanoTime() - startTime)) / 1000000l; if (remainingMs <= 0) { LOG.debug("timed out and hence returning progress log as NULL"); return new JobProgressUpdate(ProgressMonitor.NULL); } terminated = operation.waitToTerminate(Math.min(remainingMs, timeOutMs)); timeOutMs <<= 1; } } catch (InterruptedException e) { LOG.warn("Error while getting progress update", e); } ProgressMonitor pm = sessionState.getProgressMonitor(); return new JobProgressUpdate(pm != null ? pm : ProgressMonitor.NULL); }
@Before public void setup() throws Exception { HiveConf conf = new HiveConf(); conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_METRICS_ENABLED, true); MetricsFactory.init(conf); HiveSession session = mock(HiveSession.class); when(session.getHiveConf()).thenReturn(conf); when(session.getSessionState()).thenReturn(mock(SessionState.class)); when(session.getUserName()).thenReturn("userName"); operation = new SQLOperation(session, "select * from dummy", Maps.<String, String>newHashMap(), false, 0L); metrics = (CodahaleMetrics) MetricsFactory.getInstance(); }
@Test public void testQueryInfoInHookContext() throws IllegalAccessException, ClassNotFoundException, InstantiationException, HiveSQLException { HiveConf conf = new HiveConf(TestQueryHooks.class); conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); conf.set(HiveConf.ConfVars.HIVE_QUERY_LIFETIME_HOOKS.varname, QueryInfoVerificationHook.class.getName()); SessionState.start(conf); HiveSession mockHiveSession = mock(HiveSession.class); when(mockHiveSession.getHiveConf()).thenReturn(conf); when(mockHiveSession.getSessionState()).thenReturn(SessionState.get()); SQLOperation sqlOperation = new SQLOperation(mockHiveSession, QUERY, ImmutableMap.of(), false, 0); sqlOperation.run(); }
public SQLOperation(HiveSession parentSession, String statement, Map<String, String> confOverlay, boolean runInBackground, long queryTimeout) { // TODO: call setRemoteUser in ExecuteStatementOperation or higher. super(parentSession, statement, confOverlay, runInBackground); this.runAsync = runInBackground; this.queryTimeout = queryTimeout; long timeout = HiveConf.getTimeVar(queryState.getConf(), HiveConf.ConfVars.HIVE_QUERY_TIMEOUT_SECONDS, TimeUnit.SECONDS); if (timeout > 0 && (queryTimeout <= 0 || timeout < queryTimeout)) { this.queryTimeout = timeout; } this.operationLogCleanupDelayMs = HiveConf.getTimeVar(queryState.getConf(), HiveConf.ConfVars.HIVE_SERVER2_OPERATION_LOG_CLEANUP_DELAY, TimeUnit.MILLISECONDS); setupSessionIO(parentSession.getSessionState()); queryInfo = new QueryInfo(getState().toString(), getParentSession().getUserName(), getExecutionEngine(), getHandle().getHandleIdentifier().toString()); Metrics metrics = MetricsFactory.getInstance(); if (metrics != null) { submittedQryScp = metrics.createScope(MetricsConstant.HS2_SUBMITTED_QURIES); } }
private void tearDownSessionIO() { if (closeSessionStreams) { ServiceUtils.cleanup(LOG, parentSession.getSessionState().out, parentSession.getSessionState().err); } }
protected HiveCommandOperation(HiveSession parentSession, String statement, CommandProcessor commandProcessor, Map<String, String> confOverlay) { super(parentSession, statement, confOverlay, false); this.commandProcessor = commandProcessor; setupSessionIO(parentSession.getSessionState()); }
private void cleanTmpFile() { resetResultReader(); SessionState sessionState = getParentSession().getSessionState(); File tmp = sessionState.getTmpOutputFile(); tmp.delete(); }
private void cleanTmpFile() { resetResultReader(); SessionState sessionState = getParentSession().getSessionState(); sessionState.deleteTmpOutputFile(); sessionState.deleteTmpErrOutputFile(); }
private double getProgressedPercentage(OperationHandle opHandle) throws HiveSQLException { checkArgument(OperationType.EXECUTE_STATEMENT.equals(opHandle.getOperationType())); Operation operation = cliService.getSessionManager().getOperationManager().getOperation(opHandle); SessionState state = operation.getParentSession().getSessionState(); ProgressMonitor monitor = state.getProgressMonitor(); return monitor == null ? 0.0 : monitor.progressedPercentage(); }