queryStats.getCreateTime(), queryStats.getExecutionStartTime(), queryStats.getLastHeartbeat(), queryStats.getEndTime(), queryStats.getElapsedTime(), queryStats.getQueuedTime(), queryStats.getResourceWaitingTime(), queryStats.getExecutionTime(), queryStats.getAnalysisTime(), queryStats.getDistributedPlanningTime(), queryStats.getTotalPlanningTime(), queryStats.getFinishingTime(), queryStats.getTotalTasks(), queryStats.getRunningTasks(), queryStats.getCompletedTasks(), queryStats.getTotalDrivers(), queryStats.getQueuedDrivers(), queryStats.getRunningDrivers(), queryStats.getBlockedDrivers(), queryStats.getCompletedDrivers(), queryStats.getCumulativeUserMemory(), queryStats.getUserMemoryReservation(), queryStats.getTotalMemoryReservation(), queryStats.getPeakUserMemoryReservation(), queryStats.getPeakTotalMemoryReservation(), queryStats.getPeakTaskTotalMemory(), queryStats.isScheduled(), queryStats.getTotalScheduledTime(), queryStats.getTotalCpuTime(),
public BasicQueryStats(QueryStats queryStats) { this(queryStats.getCreateTime(), queryStats.getEndTime(), queryStats.getQueuedTime(), queryStats.getElapsedTime(), queryStats.getExecutionTime(), queryStats.getTotalDrivers(), queryStats.getQueuedDrivers(), queryStats.getRunningDrivers(), queryStats.getCompletedDrivers(), queryStats.getRawInputDataSize(), queryStats.getRawInputPositions(), queryStats.getCumulativeUserMemory(), queryStats.getUserMemoryReservation(), queryStats.getTotalMemoryReservation(), queryStats.getPeakUserMemoryReservation(), queryStats.getTotalCpuTime(), queryStats.getTotalScheduledTime(), queryStats.isFullyBlocked(), queryStats.getBlockedReasons(), queryStats.getProgressPercentage()); }
private QueryStatistics createQueryStatistics(QueryInfo queryInfo) { ImmutableList.Builder<String> operatorSummaries = ImmutableList.builder(); for (OperatorStats summary : queryInfo.getQueryStats().getOperatorSummaries()) { operatorSummaries.add(operatorStatsCodec.toJson(summary)); } QueryStats queryStats = queryInfo.getQueryStats(); return new QueryStatistics( ofMillis(queryStats.getTotalCpuTime().toMillis()), ofMillis(queryStats.getTotalScheduledTime().toMillis()), ofMillis(queryStats.getQueuedTime().toMillis()), Optional.of(ofMillis(queryStats.getAnalysisTime().toMillis())), Optional.of(ofMillis(queryStats.getDistributedPlanningTime().toMillis())), queryStats.getPeakUserMemoryReservation().toBytes(), queryStats.getPeakTotalMemoryReservation().toBytes(), queryStats.getPeakTaskTotalMemory().toBytes(), queryStats.getRawInputDataSize().toBytes(), queryStats.getRawInputPositions(), queryStats.getOutputDataSize().toBytes(), queryStats.getOutputPositions(), queryStats.getLogicalWrittenDataSize().toBytes(), queryStats.getWrittenPositions(), queryStats.getCumulativeUserMemory(), queryStats.getStageGcStatistics(), queryStats.getCompletedDrivers(), queryInfo.isCompleteInfo(), getCpuDistributions(queryInfo), operatorSummaries.build()); }
assertEquals(actual.getCreateTime(), new DateTime(1, UTC)); assertEquals(actual.getExecutionStartTime(), new DateTime(2, UTC)); assertEquals(actual.getLastHeartbeat(), new DateTime(3, UTC)); assertEquals(actual.getEndTime(), new DateTime(4, UTC)); assertEquals(actual.getElapsedTime(), new Duration(5, NANOSECONDS)); assertEquals(actual.getQueuedTime(), new Duration(6, NANOSECONDS)); assertEquals(actual.getAnalysisTime(), new Duration(7, NANOSECONDS)); assertEquals(actual.getDistributedPlanningTime(), new Duration(8, NANOSECONDS)); assertEquals(actual.getTotalPlanningTime(), new Duration(100, NANOSECONDS)); assertEquals(actual.getFinishingTime(), new Duration(200, NANOSECONDS)); assertEquals(actual.getTotalTasks(), 9); assertEquals(actual.getRunningTasks(), 10); assertEquals(actual.getCompletedTasks(), 11); assertEquals(actual.getTotalDrivers(), 12); assertEquals(actual.getQueuedDrivers(), 13); assertEquals(actual.getRunningDrivers(), 15); assertEquals(actual.getCompletedDrivers(), 16); assertEquals(actual.getCumulativeMemory(), 17.0); assertEquals(actual.getTotalMemoryReservation(), new DataSize(18, BYTE)); assertEquals(actual.getPeakMemoryReservation(), new DataSize(19, BYTE)); assertEquals(actual.getTotalScheduledTime(), new Duration(20, NANOSECONDS)); assertEquals(actual.getTotalCpuTime(), new Duration(21, NANOSECONDS)); assertEquals(actual.getTotalUserTime(), new Duration(22, NANOSECONDS));
private static StatementStats toStatementStats(QueryInfo queryInfo) { QueryStats queryStats = queryInfo.getQueryStats(); StageInfo outputStage = queryInfo.getOutputStage().orElse(null); return StatementStats.builder() .setState(queryInfo.getState().toString()) .setQueued(queryInfo.getState() == QueryState.QUEUED) .setScheduled(queryInfo.isScheduled()) .setNodes(globalUniqueNodes(outputStage).size()) .setTotalSplits(queryStats.getTotalDrivers()) .setQueuedSplits(queryStats.getQueuedDrivers()) .setRunningSplits(queryStats.getRunningDrivers() + queryStats.getBlockedDrivers()) .setCompletedSplits(queryStats.getCompletedDrivers()) .setCpuTimeMillis(queryStats.getTotalCpuTime().toMillis()) .setWallTimeMillis(queryStats.getTotalScheduledTime().toMillis()) .setQueuedTimeMillis(queryStats.getQueuedTime().toMillis()) .setElapsedTimeMillis(queryStats.getElapsedTime().toMillis()) .setProcessedRows(queryStats.getRawInputPositions()) .setProcessedBytes(queryStats.getRawInputDataSize().toBytes()) .setPeakMemoryBytes(queryStats.getPeakUserMemoryReservation().toBytes()) .setSpilledBytes(queryStats.getSpilledDataSize().toBytes()) .setRootStage(toStageStats(outputStage)) .build(); }
queryInfo.getFieldNames(), queryInfo.getQuery(), queryStats.getPeakMemoryReservation().toBytes(), queryStats.getCreateTime(), queryStats.getExecutionStartTime(), queryStats.getEndTime(), queryStats.getQueuedTime(), queryStats.getAnalysisTime(), queryStats.getDistributedPlanningTime(), queryStats.getTotalScheduledTime(), queryStats.getTotalCpuTime(), queryStats.getRawInputDataSize(), queryStats.getRawInputPositions(), queryStats.getTotalDrivers(), queryInfo.getErrorCode(), failureType,
queryInfo.getResourceGroupId().map(QuerySystemTable::resourceGroupIdToBlock).orElse(null), toMillis(queryStats.getQueuedTime()), toMillis(queryStats.getAnalysisTime()), toMillis(queryStats.getDistributedPlanningTime()), toTimeStamp(queryStats.getCreateTime()), toTimeStamp(queryStats.getExecutionStartTime()), toTimeStamp(queryStats.getLastHeartbeat()), toTimeStamp(queryStats.getEndTime()));
private static StatementStats toStatementStats(QueryInfo queryInfo) { QueryStats queryStats = queryInfo.getQueryStats(); return StatementStats.builder() .setState(queryInfo.getState().toString()) .setScheduled(queryInfo.isScheduled()) .setNodes(globalUniqueNodes(queryInfo.getOutputStage()).size()) .setTotalSplits(queryStats.getTotalDrivers()) .setQueuedSplits(queryStats.getQueuedDrivers()) .setRunningSplits(queryStats.getRunningDrivers()) .setCompletedSplits(queryStats.getCompletedDrivers()) .setUserTimeMillis(queryStats.getTotalUserTime().toMillis()) .setCpuTimeMillis(queryStats.getTotalCpuTime().toMillis()) .setWallTimeMillis(queryStats.getTotalScheduledTime().toMillis()) .setProcessedRows(queryStats.getRawInputPositions()) .setProcessedBytes(queryStats.getRawInputDataSize().toBytes()) .setRootStage(toStageStats(queryInfo.getOutputStage())) .build(); }
assertNotNull(queryStats.getElapsedTime()); assertNotNull(queryStats.getQueuedTime()); assertNotNull(queryStats.getResourceWaitingTime()); assertNotNull(queryStats.getExecutionTime()); assertNotNull(queryStats.getTotalPlanningTime()); assertNotNull(queryStats.getFinishingTime()); assertNotNull(queryStats.getCreateTime()); if (queryInfo.getState() == QUEUED || queryInfo.getState() == WAITING_FOR_RESOURCES) { assertNull(queryStats.getExecutionStartTime()); assertNotNull(queryStats.getExecutionStartTime()); assertNotNull(queryStats.getEndTime()); assertNull(queryStats.getEndTime());
public BasicQueryInfo(QueryInfo queryInfo) { this(queryInfo.getQueryId(), queryInfo.getSession(), queryInfo.getState(), queryInfo.getErrorType(), queryInfo.getErrorCode(), queryInfo.isScheduled(), queryInfo.getQueryStats().isFullyBlocked(), queryInfo.getQueryStats().getBlockedReasons(), queryInfo.getSelf(), queryInfo.getQuery(), queryInfo.getQueryStats().getElapsedTime(), queryInfo.getQueryStats().getEndTime(), queryInfo.getQueryStats().getCreateTime(), queryInfo.getQueryStats().getRunningDrivers(), queryInfo.getQueryStats().getQueuedDrivers(), queryInfo.getQueryStats().getCompletedDrivers(), queryInfo.getQueryStats().getTotalDrivers()); }
long rawInputBytes = info.getQueryStats().getRawInputDataSize().toBytes(); consumedCpuTimeSecs.update((long) info.getQueryStats().getTotalCpuTime().getValue(SECONDS)); consumedInputBytes.update(info.getQueryStats().getRawInputDataSize().toBytes()); consumedInputRows.update(info.getQueryStats().getRawInputPositions()); executionTime.add(info.getQueryStats().getExecutionTime()); queuedTime.add(info.getQueryStats().getQueuedTime()); long executionWallMillis = info.getQueryStats().getExecutionTime().toMillis(); if (executionWallMillis > 0) { wallInputBytesRate.add(rawInputBytes * 1000 / executionWallMillis); long executionCpuMillis = info.getQueryStats().getTotalCpuTime().toMillis(); if (executionCpuMillis > 0) { cpuInputByteRate.add(rawInputBytes * 1000 / executionCpuMillis);
assertEquals(queryStats.getElapsedTime().toMillis(), 1050); assertEquals(queryStats.getQueuedTime().toMillis(), 50); assertEquals(queryStats.getResourceWaitingTime().toMillis(), 100); assertEquals(queryStats.getTotalPlanningTime().toMillis(), 200); assertEquals(queryStats.getFinishingTime().toMillis(), 0); assertEquals(queryStats.getExecutionTime().toMillis(), 900);
assertNull(queryStats.getQueuedTime()); assertNull(queryStats.getTotalPlanningTime()); assertNull(queryStats.getExecutionStartTime()); assertNull(queryStats.getFinishingTime()); assertNull(queryStats.getEndTime()); assertNotNull(queryStats.getQueuedTime()); assertNull(queryStats.getTotalPlanningTime()); assertNull(queryStats.getExecutionStartTime()); assertNull(queryStats.getFinishingTime()); assertNull(queryStats.getEndTime()); assertNotNull(queryStats.getQueuedTime()); assertNotNull(queryStats.getTotalPlanningTime()); assertNull(queryStats.getExecutionStartTime()); assertNull(queryStats.getFinishingTime()); assertNull(queryStats.getEndTime()); assertNotNull(queryStats.getQueuedTime()); assertNotNull(queryStats.getTotalPlanningTime()); assertNotNull(queryStats.getExecutionStartTime()); assertNull(queryStats.getFinishingTime()); assertNull(queryStats.getEndTime()); assertNotNull(queryStats.getQueuedTime()); assertNotNull(queryStats.getTotalPlanningTime()); assertNotNull(queryStats.getExecutionStartTime()); assertNull(queryStats.getFinishingTime()); assertNull(queryStats.getEndTime());
DateTime queryStartTime = queryStats.getCreateTime(); DateTime queryEndTime = queryStats.getEndTime(); long planning = queryStats.getTotalPlanningTime().toMillis();
long rawInputBytes = info.getQueryStats().getRawInputDataSize().toBytes(); long executionWallMillis = info.getQueryStats().getEndTime().getMillis() - info.getQueryStats().getCreateTime().getMillis(); executionTime.add(executionWallMillis, MILLISECONDS); if (executionWallMillis > 0) { long executionCpuMillis = info.getQueryStats().getTotalCpuTime().toMillis(); if (executionCpuMillis > 0) { cpuInputByteRate.add(rawInputBytes * 1000 / executionCpuMillis);
@Test public void testWrittenStats() { String sql = "CREATE TABLE test_written_stats AS SELECT * FROM nation"; DistributedQueryRunner distributedQueryRunner = (DistributedQueryRunner) getQueryRunner(); ResultWithQueryId<MaterializedResult> resultResultWithQueryId = distributedQueryRunner.executeWithQueryId(getSession(), sql); QueryInfo queryInfo = distributedQueryRunner.getQueryInfo(resultResultWithQueryId.getQueryId()); assertEquals(queryInfo.getQueryStats().getOutputPositions(), 1L); assertEquals(queryInfo.getQueryStats().getWrittenPositions(), 25L); assertTrue(queryInfo.getQueryStats().getLogicalWrittenDataSize().toBytes() > 0L); sql = "INSERT INTO test_written_stats SELECT * FROM nation LIMIT 10"; resultResultWithQueryId = distributedQueryRunner.executeWithQueryId(getSession(), sql); queryInfo = distributedQueryRunner.getQueryInfo(resultResultWithQueryId.getQueryId()); assertEquals(queryInfo.getQueryStats().getOutputPositions(), 1L); assertEquals(queryInfo.getQueryStats().getWrittenPositions(), 10L); assertTrue(queryInfo.getQueryStats().getLogicalWrittenDataSize().toBytes() > 0L); assertUpdate("DROP TABLE test_written_stats"); }
public void queryCompletedEvent(QueryInfo queryInfo) { QueryStats queryStats = queryInfo.getQueryStats(); eventListenerManager.queryCompleted( new QueryCompletedEvent( createQueryMetadata(queryInfo), createQueryStatistics(queryInfo), createQueryContext(queryInfo.getSession(), queryInfo.getResourceGroupId()), getQueryIOMetadata(queryInfo), createQueryFailureInfo(queryInfo.getFailureInfo(), queryInfo.getOutputStage()), queryInfo.getWarnings(), ofEpochMilli(queryStats.getCreateTime().getMillis()), ofEpochMilli(queryStats.getExecutionStartTime().getMillis()), ofEpochMilli(queryStats.getEndTime() != null ? queryStats.getEndTime().getMillis() : 0))); logQueryTimeline(queryInfo); }
@Test public void testOutputStats() throws Exception { // We expect the following events // QueryCreated: 1, QueryCompleted: 1, Splits: SPLITS_PER_NODE (leaf splits) + LocalExchange[SINGLE] split + Aggregation/Output split int expectedEvents = 1 + 1 + SPLITS_PER_NODE + 1 + 1; MaterializedResult result = runQueryAndWaitForEvents("SELECT 1 FROM lineitem", expectedEvents); QueryCreatedEvent queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents()); QueryCompletedEvent queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents()); QueryStats queryStats = queryRunner.getQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getQueryStats(); assertTrue(queryStats.getOutputDataSize().toBytes() > 0L); assertTrue(queryCompletedEvent.getStatistics().getOutputBytes() > 0L); assertEquals(result.getRowCount(), queryStats.getOutputPositions()); assertEquals(result.getRowCount(), queryCompletedEvent.getStatistics().getOutputRows()); runQueryAndWaitForEvents("SELECT COUNT(1) FROM lineitem", expectedEvents); queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents()); queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents()); queryStats = queryRunner.getQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getQueryStats(); assertTrue(queryStats.getOutputDataSize().toBytes() > 0L); assertTrue(queryCompletedEvent.getStatistics().getOutputBytes() > 0L); assertEquals(1L, queryStats.getOutputPositions()); assertEquals(1L, queryCompletedEvent.getStatistics().getOutputRows()); }
@Override public DateTime getCreateTime() { return queryInfo.getQueryStats().getCreateTime(); }
@Override public Duration getTotalCpuTime() { SqlQueryScheduler scheduler = queryScheduler.get(); Optional<QueryInfo> finalQueryInfo = stateMachine.getFinalQueryInfo(); if (finalQueryInfo.isPresent()) { return finalQueryInfo.get().getQueryStats().getTotalCpuTime(); } if (scheduler == null) { return new Duration(0, SECONDS); } return scheduler.getTotalCpuTime(); }