private static void writeCheckpoint(JsonGenerator gen, AbstractCheckpointStats checkpoint) throws IOException { gen.writeNumberField("id", checkpoint.getCheckpointId()); gen.writeNumberField("trigger_timestamp", checkpoint.getTriggerTimestamp()); gen.writeNumberField("latest_ack_timestamp", checkpoint.getLatestAckTimestamp()); gen.writeNumberField("state_size", checkpoint.getStateSize()); gen.writeNumberField("end_to_end_duration", checkpoint.getEndToEndDuration()); gen.writeNumberField("alignment_buffered", checkpoint.getAlignmentBuffered()); }
/** * Try to add the checkpoint to the cache. * * @param checkpoint Checkpoint to be added. */ public void tryAdd(AbstractCheckpointStats checkpoint) { // Don't add in progress checkpoints as they will be replaced by their // completed/failed version eventually. if (cache != null && checkpoint != null && !checkpoint.getStatus().isInProgress()) { cache.put(checkpoint.getCheckpointId(), checkpoint); } }
/** * Try to add the checkpoint to the cache. * * @param checkpoint Checkpoint to be added. */ void tryAdd(AbstractCheckpointStats checkpoint) { // Don't add in progress checkpoints as they will be replaced by their // completed/failed version eventually. if (cache != null && checkpoint != null && !checkpoint.getStatus().isInProgress()) { cache.put(checkpoint.getCheckpointId(), checkpoint); } }
/** * Try to add the checkpoint to the cache. * * @param checkpoint Checkpoint to be added. */ public void tryAdd(AbstractCheckpointStats checkpoint) { // Don't add in progress checkpoints as they will be replaced by their // completed/failed version eventually. if (cache != null && checkpoint != null && !checkpoint.getStatus().isInProgress()) { cache.put(checkpoint.getCheckpointId(), checkpoint); } }
/** * Try to add the checkpoint to the cache. * * @param checkpoint Checkpoint to be added. */ public void tryAdd(AbstractCheckpointStats checkpoint) { // Don't add in progress checkpoints as they will be replaced by their // completed/failed version eventually. if (cache != null && checkpoint != null && !checkpoint.getStatus().isInProgress()) { cache.put(checkpoint.getCheckpointId(), checkpoint); } }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { String json = createCheckpointDetailsJson(checkpoint); String path = CHECKPOINT_STATS_DETAILS_REST_PATH .replace(":jobid", graph.getJobID().toString()) .replace(":checkpointid", String.valueOf(checkpoint.getCheckpointId())); archive.add(new ArchivedJson(path, json)); } return archive; } }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { String json = createCheckpointDetailsJson(checkpoint); String path = CHECKPOINT_STATS_DETAILS_REST_PATH .replace(":jobid", graph.getJobID().toString()) .replace(":checkpointid", String.valueOf(checkpoint.getCheckpointId())); archive.add(new ArchivedJson(path, json)); } return archive; } }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { for (TaskStateStats subtaskStats : checkpoint.getAllTaskStateStats()) { String json = createSubtaskCheckpointDetailsJson(checkpoint, subtaskStats); String path = CHECKPOINT_STATS_DETAILS_SUBTASKS_REST_PATH .replace(":jobid", graph.getJobID().toString()) .replace(":checkpointid", String.valueOf(checkpoint.getCheckpointId())) .replace(":vertexid", subtaskStats.getJobVertexId().toString()); archive.add(new ArchivedJson(path, json)); } } return archive; } }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size()); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { ResponseBody json = CheckpointStatistics.generateCheckpointStatistics(checkpoint, true); String path = getMessageHeaders().getTargetRestEndpointURL() .replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString()) .replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId())); archive.add(new ArchivedJson(path, json)); } return archive; } }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size()); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { ResponseBody json = CheckpointStatistics.generateCheckpointStatistics(checkpoint, true); String path = getMessageHeaders().getTargetRestEndpointURL() .replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString()) .replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId())); archive.add(new ArchivedJson(path, json)); } return archive; } }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { for (TaskStateStats subtaskStats : checkpoint.getAllTaskStateStats()) { String json = createSubtaskCheckpointDetailsJson(checkpoint, subtaskStats); String path = CHECKPOINT_STATS_DETAILS_SUBTASKS_REST_PATH .replace(":jobid", graph.getJobID().toString()) .replace(":checkpointid", String.valueOf(checkpoint.getCheckpointId())) .replace(":vertexid", subtaskStats.getJobVertexId().toString()); archive.add(new ArchivedJson(path, json)); } } return archive; } }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size()); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { ResponseBody json = CheckpointStatistics.generateCheckpointStatistics(checkpoint, true); String path = getMessageHeaders().getTargetRestEndpointURL() .replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString()) .replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId())); archive.add(new ArchivedJson(path, json)); } return archive; } }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size()); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { for (TaskStateStats subtaskStats : checkpoint.getAllTaskStateStats()) { ResponseBody json = createCheckpointDetails(checkpoint, subtaskStats); String path = getMessageHeaders().getTargetRestEndpointURL() .replace(':' + JobVertexIdPathParameter.KEY, graph.getJobID().toString()) .replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId())) .replace(':' + JobVertexIdPathParameter.KEY, subtaskStats.getJobVertexId().toString()); archive.add(new ArchivedJson(path, json)); } } return archive; }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size()); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { for (TaskStateStats subtaskStats : checkpoint.getAllTaskStateStats()) { ResponseBody json = createCheckpointDetails(checkpoint, subtaskStats); String path = getMessageHeaders().getTargetRestEndpointURL() .replace(':' + JobVertexIdPathParameter.KEY, graph.getJobID().toString()) .replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId())) .replace(':' + JobVertexIdPathParameter.KEY, subtaskStats.getJobVertexId().toString()); archive.add(new ArchivedJson(path, json)); } } return archive; }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size()); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { for (TaskStateStats subtaskStats : checkpoint.getAllTaskStateStats()) { ResponseBody json = createCheckpointDetails(checkpoint, subtaskStats); String path = getMessageHeaders().getTargetRestEndpointURL() .replace(':' + JobVertexIdPathParameter.KEY, graph.getJobID().toString()) .replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId())) .replace(':' + JobVertexIdPathParameter.KEY, subtaskStats.getJobVertexId().toString()); archive.add(new ArchivedJson(path, json)); } } return archive; }
private static void writeCheckpoint(JsonGenerator gen, AbstractCheckpointStats checkpoint) throws IOException { gen.writeNumberField(CheckpointStatistics.FIELD_NAME_ID, checkpoint.getCheckpointId()); gen.writeNumberField(CheckpointStatistics.FIELD_NAME_TRIGGER_TIMESTAMP, checkpoint.getTriggerTimestamp()); gen.writeNumberField(CheckpointStatistics.FIELD_NAME_LATEST_ACK_TIMESTAMP, checkpoint.getLatestAckTimestamp()); gen.writeNumberField(CheckpointStatistics.FIELD_NAME_STATE_SIZE, checkpoint.getStateSize()); gen.writeNumberField(CheckpointStatistics.FIELD_NAME_DURATION, checkpoint.getEndToEndDuration()); gen.writeNumberField(CheckpointStatistics.FIELD_NAME_ALIGNMENT_BUFFERED, checkpoint.getAlignmentBuffered()); }
private static TaskCheckpointStatisticsWithSubtaskDetails createCheckpointDetails(AbstractCheckpointStats checkpointStats, TaskStateStats taskStatistics) { final TaskCheckpointStatisticsWithSubtaskDetails.Summary summary = createSummary( taskStatistics.getSummaryStats(), checkpointStats.getTriggerTimestamp()); final List<SubtaskCheckpointStatistics> subtaskCheckpointStatistics = createSubtaskCheckpointStatistics( taskStatistics.getSubtaskStats(), checkpointStats.getTriggerTimestamp()); return new TaskCheckpointStatisticsWithSubtaskDetails( checkpointStats.getCheckpointId(), checkpointStats.getStatus(), taskStatistics.getLatestAckTimestamp(), taskStatistics.getStateSize(), taskStatistics.getEndToEndDuration(checkpointStats.getTriggerTimestamp()), taskStatistics.getAlignmentBuffered(), taskStatistics.getNumberOfSubtasks(), taskStatistics.getNumberOfAcknowledgedSubtasks(), summary, subtaskCheckpointStatistics); }
private static TaskCheckpointStatisticsWithSubtaskDetails createCheckpointDetails(AbstractCheckpointStats checkpointStats, TaskStateStats taskStatistics) { final TaskCheckpointStatisticsWithSubtaskDetails.Summary summary = createSummary( taskStatistics.getSummaryStats(), checkpointStats.getTriggerTimestamp()); final List<SubtaskCheckpointStatistics> subtaskCheckpointStatistics = createSubtaskCheckpointStatistics( taskStatistics.getSubtaskStats(), checkpointStats.getTriggerTimestamp()); return new TaskCheckpointStatisticsWithSubtaskDetails( checkpointStats.getCheckpointId(), checkpointStats.getStatus(), taskStatistics.getLatestAckTimestamp(), taskStatistics.getStateSize(), taskStatistics.getEndToEndDuration(checkpointStats.getTriggerTimestamp()), taskStatistics.getAlignmentBuffered(), taskStatistics.getNumberOfSubtasks(), taskStatistics.getNumberOfAcknowledgedSubtasks(), summary, subtaskCheckpointStatistics); }
private static TaskCheckpointStatisticsWithSubtaskDetails createCheckpointDetails(AbstractCheckpointStats checkpointStats, TaskStateStats taskStatistics) { final TaskCheckpointStatisticsWithSubtaskDetails.Summary summary = createSummary( taskStatistics.getSummaryStats(), checkpointStats.getTriggerTimestamp()); final List<SubtaskCheckpointStatistics> subtaskCheckpointStatistics = createSubtaskCheckpointStatistics( taskStatistics.getSubtaskStats(), checkpointStats.getTriggerTimestamp()); return new TaskCheckpointStatisticsWithSubtaskDetails( checkpointStats.getCheckpointId(), checkpointStats.getStatus(), taskStatistics.getLatestAckTimestamp(), taskStatistics.getStateSize(), taskStatistics.getEndToEndDuration(checkpointStats.getTriggerTimestamp()), taskStatistics.getAlignmentBuffered(), taskStatistics.getNumberOfSubtasks(), taskStatistics.getNumberOfAcknowledgedSubtasks(), summary, subtaskCheckpointStatistics); }
for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { gen.writeStartObject(); gen.writeNumberField(CheckpointStatistics.FIELD_NAME_ID, checkpoint.getCheckpointId()); gen.writeStringField(CheckpointStatistics.FIELD_NAME_STATUS, checkpoint.getStatus().toString()); gen.writeBooleanField(CheckpointStatistics.FIELD_NAME_IS_SAVEPOINT, checkpoint.getProperties().isSavepoint());