private void removeSafeQueryInfo(OperationHandle operationHandle) { synchronized (webuiLock) { String opKey = operationHandle.getHandleIdentifier().toString(); // remove from list of live operations QueryInfo display = liveQueryInfos.remove(opKey); if (display == null) { LOG.debug("Unexpected display object value of null for operation {}", opKey); } else if (historicalQueryInfos != null) { // add to list of saved historic operations historicalQueryInfos.put(opKey, display); } } }
@Override public void setOperationLogSessionDir(File operationLogRootDir) { if (!operationLogRootDir.exists()) { LOG.warn("The operation log root directory is removed, recreating:" + operationLogRootDir.getAbsolutePath()); if (!operationLogRootDir.mkdirs()) { LOG.warn("Unable to create operation log root directory: " + operationLogRootDir.getAbsolutePath()); } } if (!operationLogRootDir.canWrite()) { LOG.warn("The operation log root directory is not writable: " + operationLogRootDir.getAbsolutePath()); } sessionLogDir = new File(operationLogRootDir, sessionHandle.getHandleIdentifier().toString()); isOperationLogEnabled = true; if (!sessionLogDir.exists()) { if (!sessionLogDir.mkdir()) { LOG.warn("Unable to create operation log session directory: " + sessionLogDir.getAbsolutePath()); isOperationLogEnabled = false; } } if (isOperationLogEnabled) { LOG.info("Operation log session directory is created: " + sessionLogDir.getAbsolutePath()); } }
private void addOperation(Operation operation) { LOG.info("Adding operation: {} {}", operation.getHandle(), operation.getParentSession().getSessionHandle()); queryIdOperation.put(getQueryId(operation), operation); handleToOperation.put(operation.getHandle(), operation); if (operation instanceof SQLOperation) { synchronized (webuiLock) { liveQueryInfos.put(operation.getHandle().getHandleIdentifier().toString(), ((SQLOperation) operation).getQueryInfo()); } } }
public HiveSessionImpl(SessionHandle sessionHandle, TProtocolVersion protocol, String username, String password, HiveConf serverConf, String ipAddress, final List<String> forwardedAddresses) { this.username = username; this.password = password; creationTime = System.currentTimeMillis(); this.sessionHandle = sessionHandle != null ? sessionHandle : new SessionHandle(protocol); this.sessionConf = new HiveConf(serverConf); this.ipAddress = ipAddress; this.forwardedAddresses = forwardedAddresses; this.operationLock = serverConf.getBoolVar( ConfVars.HIVE_SERVER2_PARALLEL_OPS_IN_SESSION) ? null : new Semaphore(1); try { // In non-impersonation mode, map scheduler queue to current user // if fair scheduler is configured. if (! sessionConf.getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS) && sessionConf.getBoolVar(ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE)) { ShimLoader.getHadoopShims().refreshDefaultQueue(sessionConf, username); } } catch (IOException e) { LOG.warn("Error setting scheduler queue: " + e, e); } // Set an explicit session name to control the download directory name sessionConf.set(ConfVars.HIVESESSIONID.varname, this.sessionHandle.getHandleIdentifier().toString()); // Use thrift transportable formatter sessionConf.set(SerDeUtils.LIST_SINK_OUTPUT_FORMATTER, ThriftFormatter.class.getName()); sessionConf.setInt(SerDeUtils.LIST_SINK_OUTPUT_PROTOCOL, protocol.getValue()); }
public SQLOperation(HiveSession parentSession, String statement, Map<String, String> confOverlay, boolean runInBackground, long queryTimeout) { // TODO: call setRemoteUser in ExecuteStatementOperation or higher. super(parentSession, statement, confOverlay, runInBackground); this.runAsync = runInBackground; this.queryTimeout = queryTimeout; long timeout = HiveConf.getTimeVar(queryState.getConf(), HiveConf.ConfVars.HIVE_QUERY_TIMEOUT_SECONDS, TimeUnit.SECONDS); if (timeout > 0 && (queryTimeout <= 0 || timeout < queryTimeout)) { this.queryTimeout = timeout; } this.operationLogCleanupDelayMs = HiveConf.getTimeVar(queryState.getConf(), HiveConf.ConfVars.HIVE_SERVER2_OPERATION_LOG_CLEANUP_DELAY, TimeUnit.MILLISECONDS); setupSessionIO(parentSession.getSessionState()); queryInfo = new QueryInfo(getState().toString(), getParentSession().getUserName(), getExecutionEngine(), getHandle().getHandleIdentifier().toString()); Metrics metrics = MetricsFactory.getInstance(); if (metrics != null) { submittedQryScp = metrics.createScope(MetricsConstant.HS2_SUBMITTED_QURIES); } }
@Override public void setOperationLogSessionDir(File operationLogRootDir) { sessionLogDir = new File(operationLogRootDir, sessionHandle.getHandleIdentifier().toString()); isOperationLogEnabled = true; if (!sessionLogDir.exists()) { if (!sessionLogDir.mkdir()) { LOG.warn("Unable to create operation log session directory: " + sessionLogDir.getAbsolutePath()); isOperationLogEnabled = false; } } if (isOperationLogEnabled) { LOG.info("Operation log session directory is created: " + sessionLogDir.getAbsolutePath()); } }
@Override public void setOperationLogSessionDir(File operationLogRootDir) { sessionLogDir = new File(operationLogRootDir, sessionHandle.getHandleIdentifier().toString()); isOperationLogEnabled = true; if (!sessionLogDir.exists()) { if (!sessionLogDir.mkdir()) { LOG.warn("Unable to create operation log session directory: " + sessionLogDir.getAbsolutePath()); isOperationLogEnabled = false; } } if (isOperationLogEnabled) { LOG.info("Operation log session directory is created: " + sessionLogDir.getAbsolutePath()); } }
private void removeSafeQueryInfo(OperationHandle operationHandle) { synchronized (webuiLock) { String opKey = operationHandle.getHandleIdentifier().toString(); // remove from list of live operations QueryInfo display = liveQueryInfos.remove(opKey); if (display == null) { LOG.debug("Unexpected display object value of null for operation {}", opKey); } else if (historicalQueryInfos != null) { // add to list of saved historic operations historicalQueryInfos.put(opKey, display); } } }
@Override public void setOperationLogSessionDir(File operationLogRootDir) { if (!operationLogRootDir.exists()) { LOG.warn("The operation log root directory is removed, recreating:" + operationLogRootDir.getAbsolutePath()); if (!operationLogRootDir.mkdirs()) { LOG.warn("Unable to create operation log root directory: " + operationLogRootDir.getAbsolutePath()); } } if (!operationLogRootDir.canWrite()) { LOG.warn("The operation log root directory is not writable: " + operationLogRootDir.getAbsolutePath()); } sessionLogDir = new File(operationLogRootDir, sessionHandle.getHandleIdentifier().toString()); isOperationLogEnabled = true; if (!sessionLogDir.exists()) { if (!sessionLogDir.mkdir()) { LOG.warn("Unable to create operation log session directory: " + sessionLogDir.getAbsolutePath()); isOperationLogEnabled = false; } } if (isOperationLogEnabled) { LOG.info("Operation log session directory is created: " + sessionLogDir.getAbsolutePath()); } }
@Override public void setOperationLogSessionDir(File operationLogRootDir) { if (!operationLogRootDir.exists()) { LOG.warn("The operation log root directory is removed, recreating: " + operationLogRootDir.getAbsolutePath()); if (!operationLogRootDir.mkdirs()) { LOG.warn("Unable to create operation log root directory: " + operationLogRootDir.getAbsolutePath()); } } if (!operationLogRootDir.canWrite()) { LOG.warn("The operation log root directory is not writable: " + operationLogRootDir.getAbsolutePath()); } sessionLogDir = new File(operationLogRootDir, sessionHandle.getHandleIdentifier().toString()); isOperationLogEnabled = true; if (!sessionLogDir.exists()) { if (!sessionLogDir.mkdir()) { LOG.warn("Unable to create operation log session directory: " + sessionLogDir.getAbsolutePath()); isOperationLogEnabled = false; } } if (isOperationLogEnabled) { LOG.info("Operation log session directory is created: " + sessionLogDir.getAbsolutePath()); } }
@Override public void setOperationLogSessionDir(File operationLogRootDir) { if (!operationLogRootDir.exists()) { LOG.warn("The operation log root directory is removed, recreating: " + operationLogRootDir.getAbsolutePath()); if (!operationLogRootDir.mkdirs()) { LOG.warn("Unable to create operation log root directory: " + operationLogRootDir.getAbsolutePath()); } } if (!operationLogRootDir.canWrite()) { LOG.warn("The operation log root directory is not writable: " + operationLogRootDir.getAbsolutePath()); } sessionLogDir = new File(operationLogRootDir, sessionHandle.getHandleIdentifier().toString()); isOperationLogEnabled = true; if (!sessionLogDir.exists()) { if (!sessionLogDir.mkdir()) { LOG.warn("Unable to create operation log session directory: " + sessionLogDir.getAbsolutePath()); isOperationLogEnabled = false; } } if (isOperationLogEnabled) { LOG.info("Operation log session directory is created: " + sessionLogDir.getAbsolutePath()); } }
@Override public String toString() { return getSessionHandle().getHandleIdentifier().toString(); }
private void addOperation(Operation operation) { LOG.info("Adding operation: " + operation.getHandle()); queryIdOperation.put(getQueryId(operation), operation); handleToOperation.put(operation.getHandle(), operation); if (operation instanceof SQLOperation) { synchronized (webuiLock) { liveQueryInfos.put(operation.getHandle().getHandleIdentifier().toString(), ((SQLOperation) operation).getQueryInfo()); } } }
protected void createOperationLog() { if (parentSession.isOperationLogEnabled()) { File operationLogFile = new File(parentSession.getOperationLogSessionDir(), opHandle.getHandleIdentifier().toString()); isOperationLogEnabled = true;
public HiveSessionImpl(SessionHandle sessionHandle, TProtocolVersion protocol, String username, String password, HiveConf serverConf, String ipAddress, final List<String> forwardedAddresses) { this.username = username; this.password = password; creationTime = System.currentTimeMillis(); this.sessionHandle = sessionHandle != null ? sessionHandle : new SessionHandle(protocol); this.sessionConf = new HiveConf(serverConf); this.ipAddress = ipAddress; this.forwardedAddresses = forwardedAddresses; this.operationLock = serverConf.getBoolVar( ConfVars.HIVE_SERVER2_PARALLEL_OPS_IN_SESSION) ? null : new Semaphore(1); try { // In non-impersonation mode, map scheduler queue to current user // if fair scheduler is configured. if (! sessionConf.getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS) && sessionConf.getBoolVar(ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE)) { ShimLoader.getHadoopShims().refreshDefaultQueue(sessionConf, username); } } catch (IOException e) { LOG.warn("Error setting scheduler queue: " + e, e); } // Set an explicit session name to control the download directory name sessionConf.set(ConfVars.HIVESESSIONID.varname, this.sessionHandle.getHandleIdentifier().toString()); // Use thrift transportable formatter sessionConf.set(SerDeUtils.LIST_SINK_OUTPUT_FORMATTER, ThriftFormatter.class.getName()); sessionConf.setInt(SerDeUtils.LIST_SINK_OUTPUT_PROTOCOL, protocol.getValue()); }
public HiveSessionImpl(TProtocolVersion protocol, String username, String password, HiveConf serverhiveConf, String ipAddress) { this.username = username; this.password = password; this.sessionHandle = new SessionHandle(protocol); this.hiveConf = new HiveConf(serverhiveConf); this.ipAddress = ipAddress; try { // In non-impersonation mode, map scheduler queue to current user // if fair scheduler is configured. if (! hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS) && hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE)) { ShimLoader.getHadoopShims().refreshDefaultQueue(hiveConf, username); } } catch (IOException e) { LOG.warn("Error setting scheduler queue: " + e, e); } // Set an explicit session name to control the download directory name hiveConf.set(ConfVars.HIVESESSIONID.varname, sessionHandle.getHandleIdentifier().toString()); // Use thrift transportable formatter hiveConf.set(ListSinkOperator.OUTPUT_FORMATTER, FetchFormatter.ThriftFormatter.class.getName()); hiveConf.setInt(ListSinkOperator.OUTPUT_PROTOCOL, protocol.getValue()); }
public HiveSessionImpl(TProtocolVersion protocol, String username, String password, HiveConf serverhiveConf, String ipAddress) { this.username = username; this.password = password; this.sessionHandle = new SessionHandle(protocol); this.hiveConf = new HiveConf(serverhiveConf); this.ipAddress = ipAddress; try { // In non-impersonation mode, map scheduler queue to current user // if fair scheduler is configured. if (! hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS) && hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE)) { ShimLoader.getHadoopShims().refreshDefaultQueue(hiveConf, username); } } catch (IOException e) { LOG.warn("Error setting scheduler queue: " + e, e); } // Set an explicit session name to control the download directory name hiveConf.set(ConfVars.HIVESESSIONID.varname, sessionHandle.getHandleIdentifier().toString()); // Use thrift transportable formatter hiveConf.set(ListSinkOperator.OUTPUT_FORMATTER, FetchFormatter.ThriftFormatter.class.getName()); hiveConf.setInt(ListSinkOperator.OUTPUT_PROTOCOL, protocol.getValue()); }
public HiveSessionImpl(TProtocolVersion protocol, String username, String password, HiveConf serverhiveConf, String ipAddress) { this.username = username; this.password = password; this.sessionHandle = new SessionHandle(protocol); this.hiveConf = new HiveConf(serverhiveConf); this.ipAddress = ipAddress; try { // In non-impersonation mode, map scheduler queue to current user // if fair scheduler is configured. if (! hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS) && hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE)) { ShimLoader.getHadoopShims().refreshDefaultQueue(hiveConf, username); } } catch (IOException e) { LOG.warn("Error setting scheduler queue: " + e, e); } // Set an explicit session name to control the download directory name hiveConf.set(ConfVars.HIVESESSIONID.varname, sessionHandle.getHandleIdentifier().toString()); // Use thrift transportable formatter hiveConf.set(ListSinkOperator.OUTPUT_FORMATTER, FetchFormatter.ThriftFormatter.class.getName()); hiveConf.setInt(ListSinkOperator.OUTPUT_PROTOCOL, protocol.getValue()); }
public HiveSessionImpl(TProtocolVersion protocol, String username, String password, HiveConf serverhiveConf, String ipAddress) { this.username = username; this.password = password; this.sessionHandle = new SessionHandle(protocol); this.hiveConf = new HiveConf(serverhiveConf); this.ipAddress = ipAddress; try { // In non-impersonation mode, map scheduler queue to current user // if fair scheduler is configured. if (! hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_ENABLE_DOAS) && hiveConf.getBoolVar(ConfVars.HIVE_SERVER2_MAP_FAIR_SCHEDULER_QUEUE)) { ShimLoader.getHadoopShims().refreshDefaultQueue(hiveConf, username); } } catch (IOException e) { LOG.warn("Error setting scheduler queue: " + e, e); } // Set an explicit session name to control the download directory name hiveConf.set(ConfVars.HIVESESSIONID.varname, sessionHandle.getHandleIdentifier().toString()); // Use thrift transportable formatter hiveConf.set(ListSinkOperator.OUTPUT_FORMATTER, FetchFormatter.ThriftFormatter.class.getName()); hiveConf.setInt(ListSinkOperator.OUTPUT_PROTOCOL, protocol.getValue()); }
public SQLOperation(HiveSession parentSession, String statement, Map<String, String> confOverlay, boolean runInBackground, long queryTimeout) { // TODO: call setRemoteUser in ExecuteStatementOperation or higher. super(parentSession, statement, confOverlay, runInBackground); this.runAsync = runInBackground; this.queryTimeout = queryTimeout; long timeout = HiveConf.getTimeVar(queryState.getConf(), HiveConf.ConfVars.HIVE_QUERY_TIMEOUT_SECONDS, TimeUnit.SECONDS); if (timeout > 0 && (queryTimeout <= 0 || timeout < queryTimeout)) { this.queryTimeout = timeout; } this.operationLogCleanupDelayMs = HiveConf.getTimeVar(queryState.getConf(), HiveConf.ConfVars.HIVE_SERVER2_OPERATION_LOG_CLEANUP_DELAY, TimeUnit.MILLISECONDS); setupSessionIO(parentSession.getSessionState()); queryInfo = new QueryInfo(getState().toString(), getParentSession().getUserName(), getExecutionEngine(), getHandle().getHandleIdentifier().toString()); Metrics metrics = MetricsFactory.getInstance(); if (metrics != null) { submittedQryScp = metrics.createScope(MetricsConstant.HS2_SUBMITTED_QURIES); } }