for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { gen.writeStartObject(); gen.writeNumberField("id", checkpoint.getCheckpointId()); gen.writeStringField("status", checkpoint.getStatus().toString()); gen.writeBooleanField("is_savepoint", checkpoint.getProperties().isSavepoint()); gen.writeNumberField("trigger_timestamp", checkpoint.getTriggerTimestamp()); gen.writeNumberField("latest_ack_timestamp", checkpoint.getLatestAckTimestamp()); gen.writeNumberField("state_size", checkpoint.getStateSize()); gen.writeNumberField("end_to_end_duration", checkpoint.getEndToEndDuration()); gen.writeNumberField("alignment_buffered", checkpoint.getAlignmentBuffered()); gen.writeNumberField("num_subtasks", checkpoint.getNumberOfSubtasks()); gen.writeNumberField("num_acknowledged_subtasks", checkpoint.getNumberOfAcknowledgedSubtasks()); if (checkpoint.getStatus().isCompleted()) { else if (checkpoint.getStatus().isFailed()) {
/** * Returns the duration of this checkpoint calculated as the time since * triggering until the latest acknowledged subtask or <code>-1</code> if * no subtask was acknowledged yet. * * @return Duration of this checkpoint or <code>-1</code> if no subtask was acknowledged yet. */ public long getEndToEndDuration() { SubtaskStateStats subtask = getLatestAcknowledgedSubtaskStats(); if (subtask != null) { return Math.max(0, subtask.getAckTimestamp() - triggerTimestamp); } else { return -1; } }
private static void writeCheckpoint(JsonGenerator gen, AbstractCheckpointStats checkpoint) throws IOException { gen.writeNumberField("id", checkpoint.getCheckpointId()); gen.writeNumberField("trigger_timestamp", checkpoint.getTriggerTimestamp()); gen.writeNumberField("latest_ack_timestamp", checkpoint.getLatestAckTimestamp()); gen.writeNumberField("state_size", checkpoint.getStateSize()); gen.writeNumberField("end_to_end_duration", checkpoint.getEndToEndDuration()); gen.writeNumberField("alignment_buffered", checkpoint.getAlignmentBuffered()); }
/** * Try to add the checkpoint to the cache. * * @param checkpoint Checkpoint to be added. */ public void tryAdd(AbstractCheckpointStats checkpoint) { // Don't add in progress checkpoints as they will be replaced by their // completed/failed version eventually. if (cache != null && checkpoint != null && !checkpoint.getStatus().isInProgress()) { cache.put(checkpoint.getCheckpointId(), checkpoint); } }
private static TaskCheckpointStatisticsWithSubtaskDetails createCheckpointDetails(AbstractCheckpointStats checkpointStats, TaskStateStats taskStatistics) { final TaskCheckpointStatisticsWithSubtaskDetails.Summary summary = createSummary( taskStatistics.getSummaryStats(), checkpointStats.getTriggerTimestamp()); final List<SubtaskCheckpointStatistics> subtaskCheckpointStatistics = createSubtaskCheckpointStatistics( taskStatistics.getSubtaskStats(), checkpointStats.getTriggerTimestamp()); return new TaskCheckpointStatisticsWithSubtaskDetails( checkpointStats.getCheckpointId(), checkpointStats.getStatus(), taskStatistics.getLatestAckTimestamp(), taskStatistics.getStateSize(), taskStatistics.getEndToEndDuration(checkpointStats.getTriggerTimestamp()), taskStatistics.getAlignmentBuffered(), taskStatistics.getNumberOfSubtasks(), taskStatistics.getNumberOfAcknowledgedSubtasks(), summary, subtaskCheckpointStatistics); }
Collection<TaskStateStats> taskStateStats = checkpointStats.getAllTaskStateStats(); taskStateStat.getJobVertexId(), new TaskCheckpointStatistics( checkpointStats.getCheckpointId(), checkpointStats.getStatus(), taskStateStat.getLatestAckTimestamp(), taskStateStat.getStateSize(), taskStateStat.getEndToEndDuration(checkpointStats.getTriggerTimestamp()), taskStateStat.getAlignmentBuffered(), taskStateStat.getNumberOfSubtasks(),
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { for (TaskStateStats subtaskStats : checkpoint.getAllTaskStateStats()) { String json = createSubtaskCheckpointDetailsJson(checkpoint, subtaskStats); String path = CHECKPOINT_STATS_DETAILS_SUBTASKS_REST_PATH .replace(":jobid", graph.getJobID().toString()) .replace(":checkpointid", String.valueOf(checkpoint.getCheckpointId())) .replace(":vertexid", subtaskStats.getJobVertexId().toString()); archive.add(new ArchivedJson(path, json)); } } return archive; } }
checkpointsById.put(checkpoint.getCheckpointId(), checkpoint);
@Override protected TaskCheckpointStatisticsWithSubtaskDetails handleCheckpointRequest( HandlerRequest<EmptyRequestBody, TaskCheckpointMessageParameters> request, AbstractCheckpointStats checkpointStats) throws RestHandlerException { final JobVertexID jobVertexId = request.getPathParameter(JobVertexIdPathParameter.class); final TaskStateStats taskStatistics = checkpointStats.getTaskStateStats(jobVertexId); if (taskStatistics == null) { throw new NotFoundException("There is no checkpoint statistics for task " + jobVertexId + '.'); } return createCheckpointDetails(checkpointStats, taskStatistics); }
private static TaskCheckpointStatisticsWithSubtaskDetails createCheckpointDetails(AbstractCheckpointStats checkpointStats, TaskStateStats taskStatistics) { final TaskCheckpointStatisticsWithSubtaskDetails.Summary summary = createSummary( taskStatistics.getSummaryStats(), checkpointStats.getTriggerTimestamp()); final List<SubtaskCheckpointStatistics> subtaskCheckpointStatistics = createSubtaskCheckpointStatistics( taskStatistics.getSubtaskStats(), checkpointStats.getTriggerTimestamp()); return new TaskCheckpointStatisticsWithSubtaskDetails( checkpointStats.getCheckpointId(), checkpointStats.getStatus(), taskStatistics.getLatestAckTimestamp(), taskStatistics.getStateSize(), taskStatistics.getEndToEndDuration(checkpointStats.getTriggerTimestamp()), taskStatistics.getAlignmentBuffered(), taskStatistics.getNumberOfSubtasks(), taskStatistics.getNumberOfAcknowledgedSubtasks(), summary, subtaskCheckpointStatistics); }
Collection<TaskStateStats> taskStateStats = checkpointStats.getAllTaskStateStats(); taskStateStat.getJobVertexId(), new TaskCheckpointStatistics( checkpointStats.getCheckpointId(), checkpointStats.getStatus(), taskStateStat.getLatestAckTimestamp(), taskStateStat.getStateSize(), taskStateStat.getEndToEndDuration(checkpointStats.getTriggerTimestamp()), taskStateStat.getAlignmentBuffered(), taskStateStat.getNumberOfSubtasks(),
/** * Try to add the checkpoint to the cache. * * @param checkpoint Checkpoint to be added. */ void tryAdd(AbstractCheckpointStats checkpoint) { // Don't add in progress checkpoints as they will be replaced by their // completed/failed version eventually. if (cache != null && checkpoint != null && !checkpoint.getStatus().isInProgress()) { cache.put(checkpoint.getCheckpointId(), checkpoint); } }
private static void writeCheckpoint(JsonGenerator gen, AbstractCheckpointStats checkpoint) throws IOException { gen.writeNumberField(CheckpointStatistics.FIELD_NAME_ID, checkpoint.getCheckpointId()); gen.writeNumberField(CheckpointStatistics.FIELD_NAME_TRIGGER_TIMESTAMP, checkpoint.getTriggerTimestamp()); gen.writeNumberField(CheckpointStatistics.FIELD_NAME_LATEST_ACK_TIMESTAMP, checkpoint.getLatestAckTimestamp()); gen.writeNumberField(CheckpointStatistics.FIELD_NAME_STATE_SIZE, checkpoint.getStateSize()); gen.writeNumberField(CheckpointStatistics.FIELD_NAME_DURATION, checkpoint.getEndToEndDuration()); gen.writeNumberField(CheckpointStatistics.FIELD_NAME_ALIGNMENT_BUFFERED, checkpoint.getAlignmentBuffered()); }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { for (TaskStateStats subtaskStats : checkpoint.getAllTaskStateStats()) { String json = createSubtaskCheckpointDetailsJson(checkpoint, subtaskStats); String path = CHECKPOINT_STATS_DETAILS_SUBTASKS_REST_PATH .replace(":jobid", graph.getJobID().toString()) .replace(":checkpointid", String.valueOf(checkpoint.getCheckpointId())) .replace(":vertexid", subtaskStats.getJobVertexId().toString()); archive.add(new ArchivedJson(path, json)); } } return archive; } }
checkpointsById.put(checkpoint.getCheckpointId(), checkpoint);
@Override protected TaskCheckpointStatisticsWithSubtaskDetails handleCheckpointRequest( HandlerRequest<EmptyRequestBody, TaskCheckpointMessageParameters> request, AbstractCheckpointStats checkpointStats) throws RestHandlerException { final JobVertexID jobVertexId = request.getPathParameter(JobVertexIdPathParameter.class); final TaskStateStats taskStatistics = checkpointStats.getTaskStateStats(jobVertexId); if (taskStatistics == null) { throw new NotFoundException("There is no checkpoint statistics for task " + jobVertexId + '.'); } return createCheckpointDetails(checkpointStats, taskStatistics); }
gen.writeStartObject(); gen.writeNumberField("id", checkpoint.getCheckpointId()); gen.writeStringField("status", checkpoint.getStatus().toString()); gen.writeBooleanField("is_savepoint", checkpoint.getProperties().isSavepoint()); gen.writeNumberField("trigger_timestamp", checkpoint.getTriggerTimestamp()); gen.writeNumberField("latest_ack_timestamp", checkpoint.getLatestAckTimestamp()); gen.writeNumberField("state_size", checkpoint.getStateSize()); gen.writeNumberField("end_to_end_duration", checkpoint.getEndToEndDuration()); gen.writeNumberField("alignment_buffered", checkpoint.getAlignmentBuffered()); gen.writeNumberField("num_subtasks", checkpoint.getNumberOfSubtasks()); gen.writeNumberField("num_acknowledged_subtasks", checkpoint.getNumberOfAcknowledgedSubtasks()); if (checkpoint.getStatus().isCompleted()) { else if (checkpoint.getStatus().isFailed()) { for (TaskStateStats taskStats : checkpoint.getAllTaskStateStats()) { gen.writeObjectFieldStart(taskStats.getJobVertexId().toString()); gen.writeNumberField("end_to_end_duration", taskStats.getEndToEndDuration(checkpoint.getTriggerTimestamp())); gen.writeNumberField("alignment_buffered", taskStats.getAlignmentBuffered()); gen.writeNumberField("num_subtasks", taskStats.getNumberOfSubtasks());
private static TaskCheckpointStatisticsWithSubtaskDetails createCheckpointDetails(AbstractCheckpointStats checkpointStats, TaskStateStats taskStatistics) { final TaskCheckpointStatisticsWithSubtaskDetails.Summary summary = createSummary( taskStatistics.getSummaryStats(), checkpointStats.getTriggerTimestamp()); final List<SubtaskCheckpointStatistics> subtaskCheckpointStatistics = createSubtaskCheckpointStatistics( taskStatistics.getSubtaskStats(), checkpointStats.getTriggerTimestamp()); return new TaskCheckpointStatisticsWithSubtaskDetails( checkpointStats.getCheckpointId(), checkpointStats.getStatus(), taskStatistics.getLatestAckTimestamp(), taskStatistics.getStateSize(), taskStatistics.getEndToEndDuration(checkpointStats.getTriggerTimestamp()), taskStatistics.getAlignmentBuffered(), taskStatistics.getNumberOfSubtasks(), taskStatistics.getNumberOfAcknowledgedSubtasks(), summary, subtaskCheckpointStatistics); }
Collection<TaskStateStats> taskStateStats = checkpointStats.getAllTaskStateStats(); taskStateStat.getJobVertexId(), new TaskCheckpointStatistics( checkpointStats.getCheckpointId(), checkpointStats.getStatus(), taskStateStat.getLatestAckTimestamp(), taskStateStat.getStateSize(), taskStateStat.getEndToEndDuration(checkpointStats.getTriggerTimestamp()), taskStateStat.getAlignmentBuffered(), taskStateStat.getNumberOfSubtasks(),
/** * Try to add the checkpoint to the cache. * * @param checkpoint Checkpoint to be added. */ public void tryAdd(AbstractCheckpointStats checkpoint) { // Don't add in progress checkpoints as they will be replaced by their // completed/failed version eventually. if (cache != null && checkpoint != null && !checkpoint.getStatus().isInProgress()) { cache.put(checkpoint.getCheckpointId(), checkpoint); } }