@Override public Optional<DateTime> getExecutionStartTime() { return Optional.ofNullable(queryInfo.getQueryStats().getExecutionStartTime()); }
@Override public DateTime getCreateTime() { return queryInfo.getQueryStats().getCreateTime(); }
@Override public DateTime getLastHeartbeat() { return queryInfo.getQueryStats().getLastHeartbeat(); }
@Override public Optional<DateTime> getEndTime() { return Optional.ofNullable(queryInfo.getQueryStats().getEndTime()); }
@Override public Duration getTotalCpuTime() { SqlQueryScheduler scheduler = queryScheduler.get(); Optional<QueryInfo> finalQueryInfo = stateMachine.getFinalQueryInfo(); if (finalQueryInfo.isPresent()) { return finalQueryInfo.get().getQueryStats().getTotalCpuTime(); } if (scheduler == null) { return new Duration(0, SECONDS); } return scheduler.getTotalCpuTime(); }
@Override public DataSize getUserMemoryReservation() { // acquire reference to scheduler before checking finalQueryInfo, because // state change listener sets finalQueryInfo and then clears scheduler when // the query finishes. SqlQueryScheduler scheduler = queryScheduler.get(); Optional<QueryInfo> finalQueryInfo = stateMachine.getFinalQueryInfo(); if (finalQueryInfo.isPresent()) { return finalQueryInfo.get().getQueryStats().getUserMemoryReservation(); } if (scheduler == null) { return new DataSize(0, BYTE); } return succinctBytes(scheduler.getUserMemoryReservation()); }
@Override public DataSize getTotalMemoryReservation() { // acquire reference to scheduler before checking finalQueryInfo, because // state change listener sets finalQueryInfo and then clears scheduler when // the query finishes. SqlQueryScheduler scheduler = queryScheduler.get(); Optional<QueryInfo> finalQueryInfo = stateMachine.getFinalQueryInfo(); if (finalQueryInfo.isPresent()) { return finalQueryInfo.get().getQueryStats().getTotalMemoryReservation(); } if (scheduler == null) { return new DataSize(0, BYTE); } return succinctBytes(scheduler.getTotalMemoryReservation()); }
@Test public void testWrittenStats() { String sql = "CREATE TABLE test_written_stats AS SELECT * FROM nation"; DistributedQueryRunner distributedQueryRunner = (DistributedQueryRunner) getQueryRunner(); ResultWithQueryId<MaterializedResult> resultResultWithQueryId = distributedQueryRunner.executeWithQueryId(getSession(), sql); QueryInfo queryInfo = distributedQueryRunner.getQueryInfo(resultResultWithQueryId.getQueryId()); assertEquals(queryInfo.getQueryStats().getOutputPositions(), 1L); assertEquals(queryInfo.getQueryStats().getWrittenPositions(), 25L); assertTrue(queryInfo.getQueryStats().getLogicalWrittenDataSize().toBytes() > 0L); sql = "INSERT INTO test_written_stats SELECT * FROM nation LIMIT 10"; resultResultWithQueryId = distributedQueryRunner.executeWithQueryId(getSession(), sql); queryInfo = distributedQueryRunner.getQueryInfo(resultResultWithQueryId.getQueryId()); assertEquals(queryInfo.getQueryStats().getOutputPositions(), 1L); assertEquals(queryInfo.getQueryStats().getWrittenPositions(), 10L); assertTrue(queryInfo.getQueryStats().getLogicalWrittenDataSize().toBytes() > 0L); assertUpdate("DROP TABLE test_written_stats"); }
private static QueryIOMetadata getQueryIOMetadata(QueryInfo queryInfo) { ImmutableList.Builder<QueryInputMetadata> inputs = ImmutableList.builder(); for (Input input : queryInfo.getInputs()) { inputs.add(new QueryInputMetadata( input.getConnectorId().getCatalogName(), input.getSchema(), input.getTable(), input.getColumns().stream() .map(Column::getName).collect(Collectors.toList()), input.getConnectorInfo())); } Optional<QueryOutputMetadata> output = Optional.empty(); if (queryInfo.getOutput().isPresent()) { Optional<TableFinishInfo> tableFinishInfo = queryInfo.getQueryStats().getOperatorSummaries().stream() .map(OperatorStats::getInfo) .filter(TableFinishInfo.class::isInstance) .map(TableFinishInfo.class::cast) .findFirst(); output = Optional.of( new QueryOutputMetadata( queryInfo.getOutput().get().getConnectorId().getCatalogName(), queryInfo.getOutput().get().getSchema(), queryInfo.getOutput().get().getTable(), tableFinishInfo.map(TableFinishInfo::getConnectorOutputMetadata), tableFinishInfo.map(TableFinishInfo::isJsonLengthLimitExceeded))); } return new QueryIOMetadata(inputs.build(), output); }
@Override public Optional<DateTime> getEndTime() { return Optional.ofNullable(getQueryInfo().getQueryStats().getEndTime()); }
@Override public Optional<DateTime> getExecutionStartTime() { return Optional.ofNullable(getQueryInfo().getQueryStats().getExecutionStartTime()); }
@Override public DateTime getCreateTime() { return getQueryInfo().getQueryStats().getCreateTime(); }
@Override public DateTime getLastHeartbeat() { return getQueryInfo().getQueryStats().getLastHeartbeat(); }
public void queryCompletedEvent(QueryInfo queryInfo) { QueryStats queryStats = queryInfo.getQueryStats(); eventListenerManager.queryCompleted( new QueryCompletedEvent( createQueryMetadata(queryInfo), createQueryStatistics(queryInfo), createQueryContext(queryInfo.getSession(), queryInfo.getResourceGroupId()), getQueryIOMetadata(queryInfo), createQueryFailureInfo(queryInfo.getFailureInfo(), queryInfo.getOutputStage()), queryInfo.getWarnings(), ofEpochMilli(queryStats.getCreateTime().getMillis()), ofEpochMilli(queryStats.getExecutionStartTime().getMillis()), ofEpochMilli(queryStats.getEndTime() != null ? queryStats.getEndTime().getMillis() : 0))); logQueryTimeline(queryInfo); }
private QueryStatistics createQueryStatistics(QueryInfo queryInfo) { ImmutableList.Builder<String> operatorSummaries = ImmutableList.builder(); for (OperatorStats summary : queryInfo.getQueryStats().getOperatorSummaries()) { operatorSummaries.add(operatorStatsCodec.toJson(summary)); } QueryStats queryStats = queryInfo.getQueryStats(); return new QueryStatistics( ofMillis(queryStats.getTotalCpuTime().toMillis()), ofMillis(queryStats.getTotalScheduledTime().toMillis()), ofMillis(queryStats.getQueuedTime().toMillis()), Optional.of(ofMillis(queryStats.getAnalysisTime().toMillis())), Optional.of(ofMillis(queryStats.getDistributedPlanningTime().toMillis())), queryStats.getPeakUserMemoryReservation().toBytes(), queryStats.getPeakTotalMemoryReservation().toBytes(), queryStats.getPeakTaskTotalMemory().toBytes(), queryStats.getRawInputDataSize().toBytes(), queryStats.getRawInputPositions(), queryStats.getOutputDataSize().toBytes(), queryStats.getOutputPositions(), queryStats.getLogicalWrittenDataSize().toBytes(), queryStats.getWrittenPositions(), queryStats.getCumulativeUserMemory(), queryStats.getStageGcStatistics(), queryStats.getCompletedDrivers(), queryInfo.isCompleteInfo(), getCpuDistributions(queryInfo), operatorSummaries.build()); }
private void assertAllTimeSpentInQueueing(QueryState expectedState, Consumer<QueryStateMachine> stateTransition) { TestingTicker ticker = new TestingTicker(); QueryStateMachine stateMachine = createQueryStateMachineWithTicker(ticker); ticker.increment(7, MILLISECONDS); stateTransition.accept(stateMachine); assertEquals(stateMachine.getQueryState(), expectedState); QueryStats queryStats = stateMachine.getQueryInfo(Optional.empty()).getQueryStats(); assertEquals(queryStats.getQueuedTime(), new Duration(7, MILLISECONDS)); assertEquals(queryStats.getResourceWaitingTime(), new Duration(0, MILLISECONDS)); assertEquals(queryStats.getTotalPlanningTime(), new Duration(0, MILLISECONDS)); assertEquals(queryStats.getExecutionTime(), new Duration(0, MILLISECONDS)); assertEquals(queryStats.getFinishingTime(), new Duration(0, MILLISECONDS)); }
.collect(toImmutableList()); for (QueryInfo queryInfo : queryInfos) { QueryStats queryStats = queryInfo.getQueryStats(); table.addRow( queryInfo.getQueryId().toString(),
public BasicQueryInfo(QueryInfo queryInfo) { this(queryInfo.getQueryId(), queryInfo.getSession(), queryInfo.getResourceGroupId(), queryInfo.getState(), queryInfo.getMemoryPool(), queryInfo.isScheduled(), queryInfo.getSelf(), queryInfo.getQuery(), new BasicQueryStats(queryInfo.getQueryStats()), queryInfo.getErrorType(), queryInfo.getErrorCode()); }
@Test public void testOutputStats() throws Exception { // We expect the following events // QueryCreated: 1, QueryCompleted: 1, Splits: SPLITS_PER_NODE (leaf splits) + LocalExchange[SINGLE] split + Aggregation/Output split int expectedEvents = 1 + 1 + SPLITS_PER_NODE + 1 + 1; MaterializedResult result = runQueryAndWaitForEvents("SELECT 1 FROM lineitem", expectedEvents); QueryCreatedEvent queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents()); QueryCompletedEvent queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents()); QueryStats queryStats = queryRunner.getQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getQueryStats(); assertTrue(queryStats.getOutputDataSize().toBytes() > 0L); assertTrue(queryCompletedEvent.getStatistics().getOutputBytes() > 0L); assertEquals(result.getRowCount(), queryStats.getOutputPositions()); assertEquals(result.getRowCount(), queryCompletedEvent.getStatistics().getOutputRows()); runQueryAndWaitForEvents("SELECT COUNT(1) FROM lineitem", expectedEvents); queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents()); queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents()); queryStats = queryRunner.getQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getQueryStats(); assertTrue(queryStats.getOutputDataSize().toBytes() > 0L); assertTrue(queryCompletedEvent.getStatistics().getOutputBytes() > 0L); assertEquals(1L, queryStats.getOutputPositions()); assertEquals(1L, queryCompletedEvent.getStatistics().getOutputRows()); }
private static StatementStats toStatementStats(QueryInfo queryInfo) { QueryStats queryStats = queryInfo.getQueryStats(); StageInfo outputStage = queryInfo.getOutputStage().orElse(null); return StatementStats.builder() .setState(queryInfo.getState().toString()) .setQueued(queryInfo.getState() == QueryState.QUEUED) .setScheduled(queryInfo.isScheduled()) .setNodes(globalUniqueNodes(outputStage).size()) .setTotalSplits(queryStats.getTotalDrivers()) .setQueuedSplits(queryStats.getQueuedDrivers()) .setRunningSplits(queryStats.getRunningDrivers() + queryStats.getBlockedDrivers()) .setCompletedSplits(queryStats.getCompletedDrivers()) .setCpuTimeMillis(queryStats.getTotalCpuTime().toMillis()) .setWallTimeMillis(queryStats.getTotalScheduledTime().toMillis()) .setQueuedTimeMillis(queryStats.getQueuedTime().toMillis()) .setElapsedTimeMillis(queryStats.getElapsedTime().toMillis()) .setProcessedRows(queryStats.getRawInputPositions()) .setProcessedBytes(queryStats.getRawInputDataSize().toBytes()) .setPeakMemoryBytes(queryStats.getPeakUserMemoryReservation().toBytes()) .setSpilledBytes(queryStats.getSpilledDataSize().toBytes()) .setRootStage(toStageStats(outputStage)) .build(); }