queryStats.getCreateTime(), queryStats.getExecutionStartTime(), queryStats.getLastHeartbeat(), queryStats.getEndTime(), queryStats.getElapsedTime(), queryStats.getQueuedTime(), queryStats.getResourceWaitingTime(), queryStats.getExecutionTime(), queryStats.getAnalysisTime(), queryStats.getDistributedPlanningTime(), queryStats.getTotalPlanningTime(), queryStats.getFinishingTime(), queryStats.getTotalTasks(), queryStats.getRunningTasks(), queryStats.getCompletedTasks(), queryStats.getTotalDrivers(), queryStats.getQueuedDrivers(), queryStats.getRunningDrivers(), queryStats.getBlockedDrivers(), queryStats.getCompletedDrivers(), queryStats.getCumulativeUserMemory(), queryStats.getUserMemoryReservation(), queryStats.getTotalMemoryReservation(), queryStats.getPeakUserMemoryReservation(), queryStats.getPeakTotalMemoryReservation(), queryStats.getPeakTaskUserMemory(), queryStats.getPeakTaskTotalMemory(), queryStats.isScheduled(), queryStats.getTotalScheduledTime(),
private QueryStatistics createQueryStatistics(QueryInfo queryInfo) for (OperatorStats summary : queryInfo.getQueryStats().getOperatorSummaries()) { operatorSummaries.add(operatorStatsCodec.toJson(summary)); ofMillis(queryStats.getTotalCpuTime().toMillis()), ofMillis(queryStats.getTotalScheduledTime().toMillis()), ofMillis(queryStats.getQueuedTime().toMillis()), Optional.of(ofMillis(queryStats.getAnalysisTime().toMillis())), Optional.of(ofMillis(queryStats.getDistributedPlanningTime().toMillis())), queryStats.getPeakUserMemoryReservation().toBytes(), queryStats.getPeakTotalMemoryReservation().toBytes(), queryStats.getPeakTaskUserMemory().toBytes(), queryStats.getPeakTaskTotalMemory().toBytes(), queryStats.getPhysicalInputDataSize().toBytes(), queryStats.getPhysicalInputPositions(), queryStats.getInternalNetworkInputDataSize().toBytes(), queryStats.getInternalNetworkInputPositions(), queryStats.getRawInputDataSize().toBytes(), queryStats.getRawInputPositions(), queryStats.getOutputDataSize().toBytes(), queryStats.getOutputPositions(), queryStats.getLogicalWrittenDataSize().toBytes(), queryStats.getWrittenPositions(), queryStats.getCumulativeUserMemory(), queryStats.getStageGcStatistics(), queryStats.getCompletedDrivers(), queryInfo.isCompleteInfo(), getCpuDistributions(queryInfo),
public BasicQueryStats(QueryStats queryStats) { this(queryStats.getCreateTime(), queryStats.getEndTime(), queryStats.getQueuedTime(), queryStats.getElapsedTime(), queryStats.getExecutionTime(), queryStats.getTotalDrivers(), queryStats.getQueuedDrivers(), queryStats.getRunningDrivers(), queryStats.getCompletedDrivers(), queryStats.getRawInputDataSize(), queryStats.getRawInputPositions(), queryStats.getCumulativeUserMemory(), queryStats.getUserMemoryReservation(), queryStats.getTotalMemoryReservation(), queryStats.getPeakUserMemoryReservation(), queryStats.getTotalCpuTime(), queryStats.getTotalScheduledTime(), queryStats.isFullyBlocked(), queryStats.getBlockedReasons(), queryStats.getProgressPercentage()); }
private QueryStatistics createQueryStatistics(QueryInfo queryInfo) { ImmutableList.Builder<String> operatorSummaries = ImmutableList.builder(); for (OperatorStats summary : queryInfo.getQueryStats().getOperatorSummaries()) { operatorSummaries.add(operatorStatsCodec.toJson(summary)); } QueryStats queryStats = queryInfo.getQueryStats(); return new QueryStatistics( ofMillis(queryStats.getTotalCpuTime().toMillis()), ofMillis(queryStats.getTotalScheduledTime().toMillis()), ofMillis(queryStats.getQueuedTime().toMillis()), Optional.of(ofMillis(queryStats.getAnalysisTime().toMillis())), Optional.of(ofMillis(queryStats.getDistributedPlanningTime().toMillis())), queryStats.getPeakUserMemoryReservation().toBytes(), queryStats.getPeakTotalMemoryReservation().toBytes(), queryStats.getPeakTaskTotalMemory().toBytes(), queryStats.getRawInputDataSize().toBytes(), queryStats.getRawInputPositions(), queryStats.getOutputDataSize().toBytes(), queryStats.getOutputPositions(), queryStats.getLogicalWrittenDataSize().toBytes(), queryStats.getWrittenPositions(), queryStats.getCumulativeUserMemory(), queryStats.getStageGcStatistics(), queryStats.getCompletedDrivers(), queryInfo.isCompleteInfo(), getCpuDistributions(queryInfo), operatorSummaries.build()); }
private static StatementStats toStatementStats(QueryInfo queryInfo) { QueryStats queryStats = queryInfo.getQueryStats(); StageInfo outputStage = queryInfo.getOutputStage().orElse(null); return StatementStats.builder() .setState(queryInfo.getState().toString()) .setQueued(queryInfo.getState() == QueryState.QUEUED) .setScheduled(queryInfo.isScheduled()) .setNodes(globalUniqueNodes(outputStage).size()) .setTotalSplits(queryStats.getTotalDrivers()) .setQueuedSplits(queryStats.getQueuedDrivers()) .setRunningSplits(queryStats.getRunningDrivers() + queryStats.getBlockedDrivers()) .setCompletedSplits(queryStats.getCompletedDrivers()) .setCpuTimeMillis(queryStats.getTotalCpuTime().toMillis()) .setWallTimeMillis(queryStats.getTotalScheduledTime().toMillis()) .setQueuedTimeMillis(queryStats.getQueuedTime().toMillis()) .setElapsedTimeMillis(queryStats.getElapsedTime().toMillis()) .setProcessedRows(queryStats.getRawInputPositions()) .setProcessedBytes(queryStats.getRawInputDataSize().toBytes()) .setPeakMemoryBytes(queryStats.getPeakUserMemoryReservation().toBytes()) .setSpilledBytes(queryStats.getSpilledDataSize().toBytes()) .setRootStage(toStageStats(outputStage)) .build(); }
assertNotNull(queryStats.getElapsedTime()); assertNotNull(queryStats.getQueuedTime()); assertNotNull(queryStats.getResourceWaitingTime()); assertNotNull(queryStats.getExecutionTime()); assertNotNull(queryStats.getTotalPlanningTime()); assertNotNull(queryStats.getFinishingTime()); assertNotNull(queryStats.getCreateTime()); if (queryInfo.getState() == QUEUED || queryInfo.getState() == WAITING_FOR_RESOURCES) { assertNull(queryStats.getExecutionStartTime()); assertNotNull(queryStats.getExecutionStartTime()); assertNotNull(queryStats.getEndTime()); assertNull(queryStats.getEndTime());
queryInfo.getResourceGroupId().map(QuerySystemTable::resourceGroupIdToBlock).orElse(null), toMillis(queryStats.getQueuedTime()), toMillis(queryStats.getAnalysisTime()), toMillis(queryStats.getDistributedPlanningTime()), toTimeStamp(queryStats.getCreateTime()), toTimeStamp(queryStats.getExecutionStartTime()), toTimeStamp(queryStats.getLastHeartbeat()), toTimeStamp(queryStats.getEndTime()));
@Test public void testWrittenStats() { String sql = "CREATE TABLE test_written_stats AS SELECT * FROM nation"; DistributedQueryRunner distributedQueryRunner = (DistributedQueryRunner) getQueryRunner(); ResultWithQueryId<MaterializedResult> resultResultWithQueryId = distributedQueryRunner.executeWithQueryId(getSession(), sql); QueryInfo queryInfo = distributedQueryRunner.getQueryInfo(resultResultWithQueryId.getQueryId()); assertEquals(queryInfo.getQueryStats().getOutputPositions(), 1L); assertEquals(queryInfo.getQueryStats().getWrittenPositions(), 25L); assertTrue(queryInfo.getQueryStats().getLogicalWrittenDataSize().toBytes() > 0L); sql = "INSERT INTO test_written_stats SELECT * FROM nation LIMIT 10"; resultResultWithQueryId = distributedQueryRunner.executeWithQueryId(getSession(), sql); queryInfo = distributedQueryRunner.getQueryInfo(resultResultWithQueryId.getQueryId()); assertEquals(queryInfo.getQueryStats().getOutputPositions(), 1L); assertEquals(queryInfo.getQueryStats().getWrittenPositions(), 10L); assertTrue(queryInfo.getQueryStats().getLogicalWrittenDataSize().toBytes() > 0L); assertUpdate("DROP TABLE test_written_stats"); }
public void queryCompletedEvent(QueryInfo queryInfo) { QueryStats queryStats = queryInfo.getQueryStats(); eventListenerManager.queryCompleted( new QueryCompletedEvent( createQueryMetadata(queryInfo), createQueryStatistics(queryInfo), createQueryContext(queryInfo.getSession(), queryInfo.getResourceGroupId()), getQueryIOMetadata(queryInfo), createQueryFailureInfo(queryInfo.getFailureInfo(), queryInfo.getOutputStage()), queryInfo.getWarnings(), ofEpochMilli(queryStats.getCreateTime().getMillis()), ofEpochMilli(queryStats.getExecutionStartTime().getMillis()), ofEpochMilli(queryStats.getEndTime() != null ? queryStats.getEndTime().getMillis() : 0))); logQueryTimeline(queryInfo); }
DateTime queryStartTime = queryStats.getCreateTime(); DateTime queryEndTime = queryStats.getEndTime(); long planning = queryStats.getTotalPlanningTime().toMillis();
@Test public void testOutputStats() throws Exception { // We expect the following events // QueryCreated: 1, QueryCompleted: 1, Splits: SPLITS_PER_NODE (leaf splits) + LocalExchange[SINGLE] split + Aggregation/Output split int expectedEvents = 1 + 1 + SPLITS_PER_NODE + 1 + 1; MaterializedResult result = runQueryAndWaitForEvents("SELECT 1 FROM lineitem", expectedEvents); QueryCreatedEvent queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents()); QueryCompletedEvent queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents()); QueryStats queryStats = queryRunner.getQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getQueryStats(); assertTrue(queryStats.getOutputDataSize().toBytes() > 0L); assertTrue(queryCompletedEvent.getStatistics().getOutputBytes() > 0L); assertEquals(result.getRowCount(), queryStats.getOutputPositions()); assertEquals(result.getRowCount(), queryCompletedEvent.getStatistics().getOutputRows()); runQueryAndWaitForEvents("SELECT COUNT(1) FROM lineitem", expectedEvents); queryCreatedEvent = getOnlyElement(generatedEvents.getQueryCreatedEvents()); queryCompletedEvent = getOnlyElement(generatedEvents.getQueryCompletedEvents()); queryStats = queryRunner.getQueryInfo(new QueryId(queryCreatedEvent.getMetadata().getQueryId())).getQueryStats(); assertTrue(queryStats.getOutputDataSize().toBytes() > 0L); assertTrue(queryCompletedEvent.getStatistics().getOutputBytes() > 0L); assertEquals(1L, queryStats.getOutputPositions()); assertEquals(1L, queryCompletedEvent.getStatistics().getOutputRows()); }
ImmutableList.of(), "SELECT 1", new QueryStats( new DateTime(1), new DateTime(2),
@Override public DateTime getCreateTime() { return queryInfo.getQueryStats().getCreateTime(); }
public BasicQueryStats(QueryStats queryStats) { this(queryStats.getCreateTime(), queryStats.getEndTime(), queryStats.getQueuedTime(), queryStats.getElapsedTime(), queryStats.getExecutionTime(), queryStats.getTotalDrivers(), queryStats.getQueuedDrivers(), queryStats.getRunningDrivers(), queryStats.getCompletedDrivers(), queryStats.getPhysicalInputDataSize(), queryStats.getPhysicalInputPositions(), queryStats.getInternalNetworkInputDataSize(), queryStats.getInternalNetworkInputPositions(), queryStats.getRawInputDataSize(), queryStats.getRawInputPositions(), queryStats.getCumulativeUserMemory(), queryStats.getUserMemoryReservation(), queryStats.getTotalMemoryReservation(), queryStats.getPeakUserMemoryReservation(), queryStats.getTotalCpuTime(), queryStats.getTotalScheduledTime(), queryStats.isFullyBlocked(), queryStats.getBlockedReasons(), queryStats.getProgressPercentage()); }
private static StatementStats toStatementStats(QueryInfo queryInfo) { QueryStats queryStats = queryInfo.getQueryStats(); StageInfo outputStage = queryInfo.getOutputStage().orElse(null); return StatementStats.builder() .setState(queryInfo.getState().toString()) .setQueued(queryInfo.getState() == QueryState.QUEUED) .setScheduled(queryInfo.isScheduled()) .setNodes(globalUniqueNodes(outputStage).size()) .setTotalSplits(queryStats.getTotalDrivers()) .setQueuedSplits(queryStats.getQueuedDrivers()) .setRunningSplits(queryStats.getRunningDrivers() + queryStats.getBlockedDrivers()) .setCompletedSplits(queryStats.getCompletedDrivers()) .setCpuTimeMillis(queryStats.getTotalCpuTime().toMillis()) .setWallTimeMillis(queryStats.getTotalScheduledTime().toMillis()) .setQueuedTimeMillis(queryStats.getQueuedTime().toMillis()) .setElapsedTimeMillis(queryStats.getElapsedTime().toMillis()) .setProcessedRows(queryStats.getRawInputPositions()) .setProcessedBytes(queryStats.getRawInputDataSize().toBytes()) .setPeakMemoryBytes(queryStats.getPeakUserMemoryReservation().toBytes()) .setRootStage(toStageStats(outputStage)) .build(); }
assertNotNull(queryStats.getElapsedTime()); assertNotNull(queryStats.getQueuedTime()); assertNotNull(queryStats.getResourceWaitingTime()); assertNotNull(queryStats.getExecutionTime()); assertNotNull(queryStats.getTotalPlanningTime()); assertNotNull(queryStats.getFinishingTime()); assertNotNull(queryStats.getCreateTime()); if (queryInfo.getState() == QUEUED || queryInfo.getState() == WAITING_FOR_RESOURCES) { assertNull(queryStats.getExecutionStartTime()); assertNotNull(queryStats.getExecutionStartTime()); assertNotNull(queryStats.getEndTime()); assertNull(queryStats.getEndTime());
queryInfo.getResourceGroupId().map(QuerySystemTable::resourceGroupIdToBlock).orElse(null), toMillis(queryStats.getQueuedTime()), toMillis(queryStats.getAnalysisTime()), toMillis(queryStats.getDistributedPlanningTime()), toTimeStamp(queryStats.getCreateTime()), toTimeStamp(queryStats.getExecutionStartTime()), toTimeStamp(queryStats.getLastHeartbeat()), toTimeStamp(queryStats.getEndTime()));
public void queryCompletedEvent(QueryInfo queryInfo) { QueryStats queryStats = queryInfo.getQueryStats(); eventListenerManager.queryCompleted( new QueryCompletedEvent( createQueryMetadata(queryInfo), createQueryStatistics(queryInfo), createQueryContext(queryInfo.getSession(), queryInfo.getResourceGroupId()), getQueryIOMetadata(queryInfo), createQueryFailureInfo(queryInfo.getFailureInfo(), queryInfo.getOutputStage()), queryInfo.getWarnings(), ofEpochMilli(queryStats.getCreateTime().getMillis()), ofEpochMilli(queryStats.getExecutionStartTime().getMillis()), ofEpochMilli(queryStats.getEndTime() != null ? queryStats.getEndTime().getMillis() : 0))); logQueryTimeline(queryInfo); }
DateTime queryStartTime = queryStats.getCreateTime(); DateTime queryEndTime = queryStats.getEndTime(); long planning = queryStats.getTotalPlanningTime().toMillis();
ImmutableList.of(), "SELECT 1", new QueryStats( new DateTime(1), new DateTime(2),