public static ExecuteStatementOperation newExecuteStatementOperation(HiveSession parentSession, String statement, Map<String, String> confOverlay, boolean runAsync, long queryTimeout) throws HiveSQLException { String cleanStatement = HiveStringUtils.removeComments(statement); String[] tokens = cleanStatement.trim().split("\\s+"); CommandProcessor processor = null; try { processor = CommandProcessorFactory.getForHiveCommand(tokens, parentSession.getHiveConf()); } catch (SQLException e) { throw new HiveSQLException(e.getMessage(), e.getSQLState(), e); } if (processor == null) { // runAsync, queryTimeout makes sense only for a SQLOperation // Pass the original statement to SQLOperation as sql parser can remove comments by itself return new SQLOperation(parentSession, statement, confOverlay, runAsync, queryTimeout); } return new HiveCommandOperation(parentSession, cleanStatement, processor, confOverlay); } }
throws HiveSQLException { validateDefaultFetchOrientation(orientation); assertState(new ArrayList<OperationState>(Arrays.asList(OperationState.FINISHED))); RowSet rowSet = RowSetFactory.create(getResultSetSchema(), getProtocolVersion(), isBlobBased); try { driver.setMaxRows((int) maxRows); if (driver.getResults(convey)) { return decode(convey, rowSet);
@Override public void close() throws HiveSQLException { cleanup(OperationState.CLOSED); cleanupOperationLog(0); }
private RowSet decode(List<Object> rows, RowSet rowSet) throws Exception { if (driver.isFetchingTable()) { return prepareFromRow(rows, rowSet); } return decodeFromString(rows, rowSet); }
@Override public void runInternal() throws HiveSQLException { setState(OperationState.PENDING); boolean runAsync = shouldRunAsync(); final boolean asyncPrepare = runAsync && HiveConf.getBoolVar(queryState.getConf(), HiveConf.ConfVars.HIVE_SERVER2_ASYNC_EXEC_ASYNC_COMPILE); if (!asyncPrepare) { prepare(queryState); runQuery(); } else { new BackgroundWork(getCurrentUGI(), parentSession.getSessionHive(), SessionState.get(), asyncPrepare); Future<?> backgroundHandle = getParentSession().submitBackgroundOperation(work); setBackgroundHandle(backgroundHandle); } catch (RejectedExecutionException rejected) { setState(OperationState.ERROR); throw new HiveSQLException("The background threadpool cannot accept" + " new task for execution, please retry the operation", rejected);
@Override public void runInternal() throws HiveSQLException { setState(OperationState.PENDING); final HiveConf opConfig = getConfigForOperation(); prepare(opConfig); if (!shouldRunAsync()) { runQuery(opConfig); } else { final Hive parentHive = getSessionHive(); final UserGroupInformation currentUGI = getCurrentUGI(opConfig); getParentSession().getSessionManager().submitBackgroundOperation(backgroundOperation); setBackgroundHandle(backgroundHandle); } catch (RejectedExecutionException rejected) { setState(OperationState.ERROR); throw new HiveSQLException("The background threadpool cannot accept" + " new task for execution, please retry the operation", rejected);
setState(OperationState.RUNNING); try { driver = DriverFactory.newDriver(queryState, getParentSession().getUserName(), queryInfo); String guid64 = Base64.encodeBase64URLSafeString(getHandle().getHandleIdentifier() .toTHandleIdentifier().getGuid()).trim(); driver.setOperationId(guid64); throw toSQLException("Error while compiling statement", response); setHasResultSet(driver.hasResultSet()); } catch (HiveSQLException e) { setState(OperationState.ERROR); throw e; } catch (Throwable e) { setState(OperationState.ERROR); throw new HiveSQLException("Error running query: " + e.toString(), e);
private synchronized void cleanup(OperationState state) throws HiveSQLException { setState(state); if (shouldRunAsync() && state != OperationState.CANCELED && state != OperationState.TIMEDOUT) { Future<?> backgroundHandle = getBackgroundHandle(); if (backgroundHandle != null) { boolean success = backgroundHandle.cancel(true);
/** * If there are query specific settings to overlay, then create a copy of config * There are two cases we need to clone the session config that's being passed to hive driver * 1. Async query - * If the client changes a config setting, that shouldn't reflect in the execution already underway * 2. confOverlay - * The query specific settings should only be applied to the query config and not session * @return new configuration * @throws HiveSQLException */ private HiveConf getConfigForOperation() throws HiveSQLException { HiveConf sqlOperationConf = getParentSession().getHiveConf(); if (!getConfOverlay().isEmpty() || shouldRunAsync()) { // clone the parent session config for this query sqlOperationConf = new HiveConf(sqlOperationConf); // apply overlay query specific settings, if any for (Map.Entry<String, String> confEntry : getConfOverlay().entrySet()) { try { sqlOperationConf.verifyAndSet(confEntry.getKey(), confEntry.getValue()); } catch (IllegalArgumentException e) { throw new HiveSQLException("Error applying statement specific settings", e); } } } return sqlOperationConf; } }
public SQLOperation(HiveSession parentSession, String statement, Map<String, String> confOverlay, boolean runInBackground, long queryTimeout) { // TODO: call setRemoteUser in ExecuteStatementOperation or higher. super(parentSession, statement, confOverlay, runInBackground); this.runAsync = runInBackground; this.queryTimeout = queryTimeout; long timeout = HiveConf.getTimeVar(queryState.getConf(), HiveConf.ConfVars.HIVE_QUERY_TIMEOUT_SECONDS, TimeUnit.SECONDS); if (timeout > 0 && (queryTimeout <= 0 || timeout < queryTimeout)) { this.queryTimeout = timeout; } this.operationLogCleanupDelayMs = HiveConf.getTimeVar(queryState.getConf(), HiveConf.ConfVars.HIVE_SERVER2_OPERATION_LOG_CLEANUP_DELAY, TimeUnit.MILLISECONDS); setupSessionIO(parentSession.getSessionState()); queryInfo = new QueryInfo(getState().toString(), getParentSession().getUserName(), getExecutionEngine(), getHandle().getHandleIdentifier().toString()); Metrics metrics = MetricsFactory.getInstance(); if (metrics != null) { submittedQryScp = metrics.createScope(MetricsConstant.HS2_SUBMITTED_QURIES); } }
@Override public TableSchema getResultSetSchema() throws HiveSQLException { // Since compilation is always a blocking RPC call, and schema is ready after compilation, // we can return when are in the RUNNING state. assertState(Arrays.asList(OperationState.RUNNING, OperationState.FINISHED)); if (resultSchema == null) { resultSchema = new TableSchema(driver.getSchema()); } return resultSchema; }
@Override public void cancel() throws HiveSQLException { cleanup(OperationState.CANCELED); }
@Test public void testQueryInfoInHookContext() throws IllegalAccessException, ClassNotFoundException, InstantiationException, HiveSQLException { HiveConf conf = new HiveConf(TestQueryHooks.class); conf.setBoolVar(HiveConf.ConfVars.HIVE_SUPPORT_CONCURRENCY, false); conf.setVar(HiveConf.ConfVars.HIVE_AUTHORIZATION_MANAGER, "org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactory"); conf.set(HiveConf.ConfVars.HIVE_QUERY_LIFETIME_HOOKS.varname, QueryInfoVerificationHook.class.getName()); SessionState.start(conf); HiveSession mockHiveSession = mock(HiveSession.class); when(mockHiveSession.getHiveConf()).thenReturn(conf); when(mockHiveSession.getSessionState()).thenReturn(SessionState.get()); SQLOperation sqlOperation = new SQLOperation(mockHiveSession, QUERY, ImmutableMap.of(), false, 0); sqlOperation.run(); }
private void registerCurrentOperationLog() { if (isOperationLogEnabled) { if (operationLog == null) { LOG.warn("Failed to get current OperationLog object of Operation: " + getHandle().getHandleIdentifier()); isOperationLogEnabled = false; return; } OperationLog.setCurrentOperationLog(operationLog); } }
@Override public void runInternal() throws HiveSQLException { setState(OperationState.PENDING); final HiveConf opConfig = getConfigForOperation(); prepare(opConfig); if (!shouldRunAsync()) { runQuery(opConfig); } else { final Hive parentHive = getSessionHive(); final UserGroupInformation currentUGI = getCurrentUGI(opConfig); getParentSession().getSessionManager().submitBackgroundOperation(backgroundOperation); setBackgroundHandle(backgroundHandle); } catch (RejectedExecutionException rejected) { setState(OperationState.ERROR); throw new HiveSQLException("The background threadpool cannot accept" + " new task for execution, please retry the operation", rejected);
setState(OperationState.RUNNING); try { driver = DriverFactory.newDriver(queryState, getParentSession().getUserName(), queryInfo); String guid64 = Base64.encodeBase64URLSafeString(getHandle().getHandleIdentifier() .toTHandleIdentifier().getGuid()).trim(); driver.setOperationId(guid64); throw toSQLException("Error while compiling statement", response); setHasResultSet(driver.hasResultSet()); } catch (HiveSQLException e) { setState(OperationState.ERROR); throw e; } catch (Throwable e) { setState(OperationState.ERROR); throw new HiveSQLException("Error running query: " + e.toString(), e);
@Override public void runInternal() throws HiveSQLException { setState(OperationState.PENDING); boolean runAsync = shouldRunAsync(); final boolean asyncPrepare = runAsync && HiveConf.getBoolVar(queryState.getConf(), HiveConf.ConfVars.HIVE_SERVER2_ASYNC_EXEC_ASYNC_COMPILE); if (!asyncPrepare) { prepare(queryState); } if (!runAsync) { runQuery(); } else { // We'll pass ThreadLocals in the background thread from the foreground (handler) thread. // 1) ThreadLocal Hive object needs to be set in background thread // 2) The metastore client in Hive is associated with right user. // 3) Current UGI will get used by metastore when metastore is in embedded mode Runnable work = new BackgroundWork(getCurrentUGI(), parentSession.getSessionHive(), SessionState.getPerfLogger(), SessionState.get(), asyncPrepare); try { // This submit blocks if no background threads are available to run this operation Future<?> backgroundHandle = getParentSession().submitBackgroundOperation(work); setBackgroundHandle(backgroundHandle); } catch (RejectedExecutionException rejected) { setState(OperationState.ERROR); throw new HiveSQLException("The background threadpool cannot accept" + " new task for execution, please retry the operation", rejected); } } }
private void cleanup(OperationState state) throws HiveSQLException { setState(state); if (shouldRunAsync()) { Future<?> backgroundHandle = getBackgroundHandle(); if (backgroundHandle != null) { backgroundHandle.cancel(true); } } if (driver != null) { driver.close(); driver.destroy(); } driver = null; SessionState ss = SessionState.get(); if (ss.getTmpOutputFile() != null) { ss.getTmpOutputFile().delete(); } }
/** * If there are query specific settings to overlay, then create a copy of config * There are two cases we need to clone the session config that's being passed to hive driver * 1. Async query - * If the client changes a config setting, that shouldn't reflect in the execution already underway * 2. confOverlay - * The query specific settings should only be applied to the query config and not session * @return new configuration * @throws HiveSQLException */ private HiveConf getConfigForOperation() throws HiveSQLException { HiveConf sqlOperationConf = getParentSession().getHiveConf(); if (!getConfOverlay().isEmpty() || shouldRunAsync()) { // clone the parent session config for this query sqlOperationConf = new HiveConf(sqlOperationConf); // apply overlay query specific settings, if any for (Map.Entry<String, String> confEntry : getConfOverlay().entrySet()) { try { sqlOperationConf.verifyAndSet(confEntry.getKey(), confEntry.getValue()); } catch (IllegalArgumentException e) { throw new HiveSQLException("Error applying statement specific settings", e); } } } return sqlOperationConf; } }
private RowSet decode(List<Object> rows, RowSet rowSet) throws Exception { if (driver.isFetchingTable()) { return prepareFromRow(rows, rowSet); } return decodeFromString(rows, rowSet); }