private static QueryIOMetadata getQueryIOMetadata(QueryInfo queryInfo) { ImmutableList.Builder<QueryInputMetadata> inputs = ImmutableList.builder(); for (Input input : queryInfo.getInputs()) { inputs.add(new QueryInputMetadata( input.getConnectorId().getCatalogName(), input.getSchema(), input.getTable(), input.getColumns().stream() .map(Column::getName).collect(Collectors.toList()), input.getConnectorInfo())); } Optional<QueryOutputMetadata> output = Optional.empty(); if (queryInfo.getOutput().isPresent()) { Optional<TableFinishInfo> tableFinishInfo = queryInfo.getQueryStats().getOperatorSummaries().stream() .map(OperatorStats::getInfo) .filter(TableFinishInfo.class::isInstance) .map(TableFinishInfo.class::cast) .findFirst(); output = Optional.of( new QueryOutputMetadata( queryInfo.getOutput().get().getConnectorId().getCatalogName(), queryInfo.getOutput().get().getSchema(), queryInfo.getOutput().get().getTable(), tableFinishInfo.map(TableFinishInfo::getConnectorOutputMetadata), tableFinishInfo.map(TableFinishInfo::isJsonLengthLimitExceeded))); } return new QueryIOMetadata(inputs.build(), output); }
if (!finalInfo.isPresent() || !finalInfo.get().getOutputStage().isPresent()) { return; Optional<StageInfo> prunedOutputStage = queryInfo.getOutputStage().map(outputStage -> new StageInfo( outputStage.getStageId(), outputStage.getState(), queryInfo.getQueryId(), queryInfo.getSession(), queryInfo.getState(), getMemoryPool().getId(), queryInfo.isScheduled(), queryInfo.getSelf(), queryInfo.getFieldNames(), queryInfo.getQuery(), pruneQueryStats(queryInfo.getQueryStats()), queryInfo.getSetCatalog(), queryInfo.getSetSchema(), queryInfo.getSetPath(), queryInfo.getSetSessionProperties(), queryInfo.getResetSessionProperties(), queryInfo.getAddedPreparedStatements(), queryInfo.getDeallocatedPreparedStatements(), queryInfo.getStartedTransactionId(), queryInfo.isClearTransactionId(), queryInfo.getUpdateType(), prunedOutputStage, queryInfo.getFailureInfo(), queryInfo.getErrorCode(),
public BasicQueryInfo(QueryInfo queryInfo) { this(queryInfo.getQueryId(), queryInfo.getSession(), queryInfo.getResourceGroupId(), queryInfo.getState(), queryInfo.getMemoryPool(), queryInfo.isScheduled(), queryInfo.getSelf(), queryInfo.getQuery(), new BasicQueryStats(queryInfo.getQueryStats()), queryInfo.getErrorType(), queryInfo.getErrorCode()); }
private synchronized void closeExchangeClientIfNecessary(QueryInfo queryInfo) { // Close the exchange client if the query has failed, or if the query // is done and it does not have an output stage. The latter happens // for data definition executions, as those do not have output. if ((queryInfo.getState() == FAILED) || (queryInfo.getState().isDone() && !queryInfo.getOutputStage().isPresent())) { exchangeClient.close(); } }
private QueryMetadata createQueryMetadata(QueryInfo queryInfo) { return new QueryMetadata( queryInfo.getQueryId().toString(), queryInfo.getSession().getTransactionId().map(TransactionId::toString), queryInfo.getQuery(), queryInfo.getState().toString(), queryInfo.getSelf(), createTextQueryPlan(queryInfo), queryInfo.getOutputStage().flatMap(stage -> stageInfoCodec.toJsonWithLengthLimit(stage, maxJsonLimit))); }
private Optional<String> createTextQueryPlan(QueryInfo queryInfo) { try { if (queryInfo.getOutputStage().isPresent()) { return Optional.of(textDistributedPlan( queryInfo.getOutputStage().get(), functionRegistry, queryInfo.getSession().toSession(sessionPropertyManager), false)); } } catch (Exception e) { // Sometimes it is expected to fail. For example if generated plan is too long. // Don't fail to create event if the plan can not be created. log.warn(e, "Error creating explain plan for query %s", queryInfo.getQueryId()); } return Optional.empty(); }
@Override public Duration getTotalCpuTime() { SqlQueryScheduler scheduler = queryScheduler.get(); Optional<QueryInfo> finalQueryInfo = stateMachine.getFinalQueryInfo(); if (finalQueryInfo.isPresent()) { return finalQueryInfo.get().getQueryStats().getTotalCpuTime(); } if (scheduler == null) { return new Duration(0, SECONDS); } return scheduler.getTotalCpuTime(); }
private static List<StageCpuDistribution> getCpuDistributions(QueryInfo queryInfo) { if (!queryInfo.getOutputStage().isPresent()) { return ImmutableList.of(); } ImmutableList.Builder<StageCpuDistribution> builder = ImmutableList.builder(); populateDistribution(queryInfo.getOutputStage().get(), builder); return builder.build(); }
QueryInfo queryInfo = stateMachine.getQueryInfo(Optional.empty()); assertEquals(queryInfo.getQueryId(), TEST_SESSION.getQueryId()); assertEquals(queryInfo.getSelf(), LOCATION); assertFalse(queryInfo.getOutputStage().isPresent()); assertEquals(queryInfo.getQuery(), QUERY); assertEquals(queryInfo.getInputs(), INPUTS); assertEquals(queryInfo.getOutput(), OUTPUT); assertEquals(queryInfo.getFieldNames(), OUTPUT_FIELD_NAMES); assertEquals(queryInfo.getUpdateType(), UPDATE_TYPE); assertEquals(queryInfo.getMemoryPool(), MEMORY_POOL.getId()); QueryStats queryStats = queryInfo.getQueryStats(); if (queryInfo.getState() == QUEUED || queryInfo.getState() == WAITING_FOR_RESOURCES) { if (queryInfo.getState().isDone()) { assertEquals(queryInfo.getState(), expectedState); assertNotNull(queryInfo.getFailureInfo()); FailureInfo failure = queryInfo.getFailureInfo().toFailureInfo(); if (expectedException instanceof PrestoException) { assertEquals(queryInfo.getErrorCode(), ((PrestoException) expectedException).getErrorCode()); assertEquals(queryInfo.getErrorCode(), GENERIC_INTERNAL_ERROR.toErrorCode()); assertNull(queryInfo.getFailureInfo());
private static void logQueryTimeline(QueryInfo queryInfo) QueryStats queryStats = queryInfo.getQueryStats(); DateTime queryStartTime = queryStats.getCreateTime(); DateTime queryEndTime = queryStats.getEndTime(); List<StageInfo> stages = StageInfo.getAllStages(queryInfo.getOutputStage()); queryInfo.getQueryId(), queryInfo.getSession().getTransactionId().map(TransactionId::toString).orElse(""), elapsed, planning,
private static URI findCancelableLeafStage(QueryInfo queryInfo) { // if query is running, find the leaf-most running stage return queryInfo.getOutputStage().map(Query::findCancelableLeafStage).orElse(null); }
session = queryManager.getFullQueryInfo(queryId).getSession().toSession(sessionPropertyManager); serde = new PagesSerdeFactory(blockEncodingSerde, isExchangeCompressionEnabled(session)).createPagesSerde(); if ((data != null) && (queryInfo.getUpdateType() != null) && (updateCount == null) && (columns.size() == 1) && (columns.get(0).getType().equals(StandardTypes.BIGINT))) { Iterator<List<Object>> iterator = data.iterator(); if ((queryInfo.getState() == QueryState.FINISHED) && !queryInfo.getOutputStage().isPresent()) { columns = ImmutableList.of(new Column("result", BooleanType.BOOLEAN)); data = ImmutableSet.of(ImmutableList.of(true)); if (!queryInfo.isFinalQueryInfo() && !queryInfo.getState().equals(QueryState.FAILED) || !exchangeClient.isClosed()) { nextResultsUri = createNextResultsUri(scheme, uriInfo); setCatalog = queryInfo.getSetCatalog(); setSchema = queryInfo.getSetSchema(); setPath = queryInfo.getSetPath(); setSessionProperties = queryInfo.getSetSessionProperties(); resetSessionProperties = queryInfo.getResetSessionProperties(); addedPreparedStatements = queryInfo.getAddedPreparedStatements(); deallocatedPreparedStatements = queryInfo.getDeallocatedPreparedStatements(); startedTransactionId = queryInfo.getStartedTransactionId(); clearTransactionId = queryInfo.isClearTransactionId(); toStatementStats(queryInfo),
@Override public Optional<ErrorCode> getErrorCode() { return Optional.ofNullable(getQueryInfo().getFailureInfo()).map(ExecutionFailureInfo::getErrorCode); }
QueryStats queryStats = queryInfo.getQueryStats(); FailureInfo failureInfo = queryInfo.getFailureInfo(); mergedProperties.putAll(queryInfo.getSession().getSystemProperties()); for (Map.Entry<String, Map<String, String>> catalogEntry : queryInfo.getSession().getCatalogProperties().entrySet()) { for (Map.Entry<String, String> entry : catalogEntry.getValue().entrySet()) { mergedProperties.put(catalogEntry.getKey() + "." + entry.getKey(), entry.getValue()); Optional<TaskInfo> task = findFailedTask(queryInfo.getOutputStage()); String failureHost = task.map(x -> x.getSelf().getHost()).orElse(null); String failureTask = task.map(x -> x.getTaskId().toString()).orElse(null); queryInfo.getQueryId(), queryInfo.getSession().getTransactionId().map(TransactionId::toString).orElse(null), queryInfo.getSession().getUser(), queryInfo.getSession().getPrincipal().orElse(null), queryInfo.getSession().getSource().orElse(null), serverVersion, environment, queryInfo.getSession().getCatalog().orElse(null), queryInfo.getSession().getSchema().orElse(null), queryInfo.getSession().getRemoteUserAddress().orElse(null), queryInfo.getSession().getUserAgent().orElse(null), queryInfo.getState(), queryInfo.getSelf(), queryInfo.getFieldNames(), queryInfo.getQuery(), queryStats.getPeakMemoryReservation().toBytes(), queryStats.getCreateTime(),
@Test public void testNonLeafGroup() throws Exception { Session session = testSessionBuilder() .setCatalog("tpch") .setSchema("sf100000") .setSource("non-leaf") .build(); QueryManager queryManager = queryRunner.getCoordinator().getQueryManager(); InternalResourceGroupManager manager = queryRunner.getCoordinator().getResourceGroupManager().get(); DbResourceGroupConfigurationManager dbConfigurationManager = (DbResourceGroupConfigurationManager) manager.getConfigurationManager(); int originalSize = getSelectors(queryRunner).size(); // Add a selector for a non leaf group dao.insertSelector(3, 100, "user.*", "(?i).*non-leaf.*", null, null, null); dbConfigurationManager.load(); while (getSelectors(queryRunner).size() != originalSize + 1) { MILLISECONDS.sleep(500); } // Submit query with side effect of creating resource groups QueryId firstDashboardQuery = createQuery(queryRunner, dashboardSession(), LONG_LASTING_QUERY); waitForQueryState(queryRunner, firstDashboardQuery, RUNNING); cancelQuery(queryRunner, firstDashboardQuery); waitForQueryState(queryRunner, firstDashboardQuery, FAILED); // Submit a query to a non-leaf resource group QueryId invalidResourceGroupQuery = createQuery(queryRunner, session, LONG_LASTING_QUERY); waitForQueryState(queryRunner, invalidResourceGroupQuery, FAILED); assertEquals(queryRunner.getQueryInfo(invalidResourceGroupQuery).getErrorCode(), INVALID_RESOURCE_GROUP.toErrorCode()); }
List<QueryInfo> queryInfos = queryManager.getQueries().stream() .map(BasicQueryInfo::getQueryId) .map(queryId -> { try { return queryManager.getFullQueryInfo(queryId); .collect(toImmutableList()); for (QueryInfo queryInfo : queryInfos) { QueryStats queryStats = queryInfo.getQueryStats(); table.addRow( queryInfo.getQueryId().toString(), queryInfo.getState().toString(), queryInfo.getSession().getUser(), queryInfo.getSession().getSource().orElse(null), queryInfo.getQuery(), queryInfo.getResourceGroupId().map(QuerySystemTable::resourceGroupIdToBlock).orElse(null),
@Override public Optional<DateTime> getExecutionStartTime() { return Optional.ofNullable(queryInfo.getQueryStats().getExecutionStartTime()); }
public void createdEvent(QueryInfo queryInfo) { eventClient.post( new QueryCreatedEvent( queryInfo.getQueryId(), queryInfo.getSession().getTransactionId().map(TransactionId::toString).orElse(null), queryInfo.getSession().getUser(), queryInfo.getSession().getPrincipal().orElse(null), queryInfo.getSession().getSource().orElse(null), serverVersion, environment, queryInfo.getSession().getCatalog().orElse(null), queryInfo.getSession().getSchema().orElse(null), queryInfo.getSession().getRemoteUserAddress().orElse(null), queryInfo.getSession().getUserAgent().orElse(null), queryInfo.getSelf(), queryInfo.getQuery(), queryInfo.getQueryStats().getCreateTime() ) ); }
@Test(timeOut = SHUTDOWN_TIMEOUT_MILLIS) public void testShutdown() throws Exception .filter(server -> !server.isCoordinator()) .findFirst() .get(); TaskManager taskManager = worker.getTaskManager(); worker.getGracefulShutdownHandler().requestShutdown(); List<QueryInfo> queryInfos = queryRunner.getCoordinator().getQueryManager().getAllQueryInfo(); for (QueryInfo info : queryInfos) { assertEquals(info.getState(), FINISHED); TestShutdownAction shutdownAction = (TestShutdownAction) worker.getShutdownAction(); shutdownAction.waitForShutdownComplete(SHUTDOWN_TIMEOUT_MILLIS); assertTrue(shutdownAction.isWorkerShutdown());
private static StatementStats toStatementStats(QueryInfo queryInfo) { QueryStats queryStats = queryInfo.getQueryStats(); StageInfo outputStage = queryInfo.getOutputStage().orElse(null); return StatementStats.builder() .setState(queryInfo.getState().toString()) .setQueued(queryInfo.getState() == QueryState.QUEUED) .setScheduled(queryInfo.isScheduled()) .setNodes(globalUniqueNodes(outputStage).size()) .setTotalSplits(queryStats.getTotalDrivers()) .setQueuedSplits(queryStats.getQueuedDrivers()) .setRunningSplits(queryStats.getRunningDrivers() + queryStats.getBlockedDrivers()) .setCompletedSplits(queryStats.getCompletedDrivers()) .setCpuTimeMillis(queryStats.getTotalCpuTime().toMillis()) .setWallTimeMillis(queryStats.getTotalScheduledTime().toMillis()) .setQueuedTimeMillis(queryStats.getQueuedTime().toMillis()) .setElapsedTimeMillis(queryStats.getElapsedTime().toMillis()) .setProcessedRows(queryStats.getRawInputPositions()) .setProcessedBytes(queryStats.getRawInputDataSize().toBytes()) .setPeakMemoryBytes(queryStats.getPeakUserMemoryReservation().toBytes()) .setSpilledBytes(queryStats.getSpilledDataSize().toBytes()) .setRootStage(toStageStats(outputStage)) .build(); }