/** * Creates a new snapshot of the available stats. * * @return The latest statistics snapshot. */ public CheckpointStatsSnapshot createSnapshot() { CheckpointStatsSnapshot snapshot = latestSnapshot; // Only create a new snapshot if dirty and no update in progress, // because we don't want to block the coordinator. if (dirty && statsReadWriteLock.tryLock()) { try { // Create a new snapshot snapshot = new CheckpointStatsSnapshot( counts.createSnapshot(), summary.createSnapshot(), history.createSnapshot(), latestRestoredCheckpoint); latestSnapshot = snapshot; dirty = false; } finally { statsReadWriteLock.unlock(); } } return snapshot; }
private static String createCheckpointStatsJson(AccessExecutionGraph graph) throws IOException { StringWriter writer = new StringWriter(); JsonGenerator gen = JsonFactory.jacksonFactory.createGenerator(writer); CheckpointStatsSnapshot snapshot = graph.getCheckpointStatsSnapshot(); if (snapshot == null) { return "{}"; } gen.writeStartObject(); // Counts writeCounts(gen, snapshot.getCounts()); // Summary writeSummary(gen, snapshot.getSummaryStats()); CheckpointStatsHistory history = snapshot.getHistory(); // Latest writeLatestCheckpoints( gen, history.getLatestCompletedCheckpoint(), history.getLatestSavepoint(), history.getLatestFailedCheckpoint(), snapshot.getLatestRestoredCheckpoint()); // History writeHistory(gen, snapshot.getHistory()); gen.writeEndObject(); gen.close(); return writer.toString(); }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { String json = createCheckpointDetailsJson(checkpoint); String path = CHECKPOINT_STATS_DETAILS_REST_PATH .replace(":jobid", graph.getJobID().toString()) .replace(":checkpointid", String.valueOf(checkpoint.getCheckpointId())); archive.add(new ArchivedJson(path, json)); } return archive; } }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { String json = createCheckpointDetailsJson(checkpoint); String path = CHECKPOINT_STATS_DETAILS_REST_PATH .replace(":jobid", graph.getJobID().toString()) .replace(":checkpointid", String.valueOf(checkpoint.getCheckpointId())); archive.add(new ArchivedJson(path, json)); } return archive; } }
private static String createCheckpointStatsJson(AccessExecutionGraph graph) throws IOException { StringWriter writer = new StringWriter(); JsonGenerator gen = JsonFactory.JACKSON_FACTORY.createGenerator(writer); CheckpointStatsSnapshot snapshot = graph.getCheckpointStatsSnapshot(); if (snapshot == null) { return "{}"; } gen.writeStartObject(); // Counts writeCounts(gen, snapshot.getCounts()); // Summary writeSummary(gen, snapshot.getSummaryStats()); CheckpointStatsHistory history = snapshot.getHistory(); // Latest writeLatestCheckpoints( gen, history.getLatestCompletedCheckpoint(), history.getLatestSavepoint(), history.getLatestFailedCheckpoint(), snapshot.getLatestRestoredCheckpoint()); // History writeHistory(gen, snapshot.getHistory()); gen.writeEndObject(); gen.close(); return writer.toString(); }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size()); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { ResponseBody json = CheckpointStatistics.generateCheckpointStatistics(checkpoint, true); String path = getMessageHeaders().getTargetRestEndpointURL() .replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString()) .replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId())); archive.add(new ArchivedJson(path, json)); } return archive; } }
/** * Creates a new snapshot of the available stats. * * @return The latest statistics snapshot. */ public CheckpointStatsSnapshot createSnapshot() { CheckpointStatsSnapshot snapshot = latestSnapshot; // Only create a new snapshot if dirty and no update in progress, // because we don't want to block the coordinator. if (dirty && statsReadWriteLock.tryLock()) { try { // Create a new snapshot snapshot = new CheckpointStatsSnapshot( counts.createSnapshot(), summary.createSnapshot(), history.createSnapshot(), latestRestoredCheckpoint); latestSnapshot = snapshot; dirty = false; } finally { statsReadWriteLock.unlock(); } } return snapshot; }
throw new RestHandlerException("Checkpointing has not been enabled.", HttpResponseStatus.NOT_FOUND); } else { final CheckpointStatsCounts checkpointStatsCounts = checkpointStatsSnapshot.getCounts(); checkpointStatsCounts.getNumberOfFailedCheckpoints()); final CompletedCheckpointStatsSummary checkpointStatsSummary = checkpointStatsSnapshot.getSummaryStats(); final MinMaxAvgStats stateSize = checkpointStatsSummary.getStateSizeStats(); final MinMaxAvgStats duration = checkpointStatsSummary.getEndToEndDurationStats(); alignment.getAverage())); final CheckpointStatsHistory checkpointStatsHistory = checkpointStatsSnapshot.getHistory(); null; final RestoredCheckpointStats restoredCheckpointStats = checkpointStatsSnapshot.getLatestRestoredCheckpoint(); for (AbstractCheckpointStats abstractCheckpointStats : checkpointStatsSnapshot.getHistory().getCheckpoints()) { history.add(CheckpointStatistics.generateCheckpointStatistics(abstractCheckpointStats, false));
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size()); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { ResponseBody json = CheckpointStatistics.generateCheckpointStatistics(checkpoint, true); String path = getMessageHeaders().getTargetRestEndpointURL() .replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString()) .replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId())); archive.add(new ArchivedJson(path, json)); } return archive; } }
/** * Creates a new snapshot of the available stats. * * @return The latest statistics snapshot. */ public CheckpointStatsSnapshot createSnapshot() { CheckpointStatsSnapshot snapshot = latestSnapshot; // Only create a new snapshot if dirty and no update in progress, // because we don't want to block the coordinator. if (dirty && statsReadWriteLock.tryLock()) { try { // Create a new snapshot snapshot = new CheckpointStatsSnapshot( counts.createSnapshot(), summary.createSnapshot(), history.createSnapshot(), latestRestoredCheckpoint); latestSnapshot = snapshot; dirty = false; } finally { statsReadWriteLock.unlock(); } } return snapshot; }
throw new RestHandlerException("Checkpointing has not been enabled.", HttpResponseStatus.NOT_FOUND); } else { final CheckpointStatsCounts checkpointStatsCounts = checkpointStatsSnapshot.getCounts(); checkpointStatsCounts.getNumberOfFailedCheckpoints()); final CompletedCheckpointStatsSummary checkpointStatsSummary = checkpointStatsSnapshot.getSummaryStats(); final MinMaxAvgStats stateSize = checkpointStatsSummary.getStateSizeStats(); final MinMaxAvgStats duration = checkpointStatsSummary.getEndToEndDurationStats(); alignment.getAverage())); final CheckpointStatsHistory checkpointStatsHistory = checkpointStatsSnapshot.getHistory(); null; final RestoredCheckpointStats restoredCheckpointStats = checkpointStatsSnapshot.getLatestRestoredCheckpoint(); for (AbstractCheckpointStats abstractCheckpointStats : checkpointStatsSnapshot.getHistory().getCheckpoints()) { history.add(CheckpointStatistics.generateCheckpointStatistics(abstractCheckpointStats, false));
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { for (TaskStateStats subtaskStats : checkpoint.getAllTaskStateStats()) { String json = createSubtaskCheckpointDetailsJson(checkpoint, subtaskStats); String path = CHECKPOINT_STATS_DETAILS_SUBTASKS_REST_PATH .replace(":jobid", graph.getJobID().toString()) .replace(":checkpointid", String.valueOf(checkpoint.getCheckpointId())) .replace(":vertexid", subtaskStats.getJobVertexId().toString()); archive.add(new ArchivedJson(path, json)); } } return archive; } }
/** * Creates a new snapshot of the available stats. * * @return The latest statistics snapshot. */ public CheckpointStatsSnapshot createSnapshot() { CheckpointStatsSnapshot snapshot = latestSnapshot; // Only create a new snapshot if dirty and no update in progress, // because we don't want to block the coordinator. if (dirty && statsReadWriteLock.tryLock()) { try { // Create a new snapshot snapshot = new CheckpointStatsSnapshot( counts.createSnapshot(), summary.createSnapshot(), history.createSnapshot(), latestRestoredCheckpoint); latestSnapshot = snapshot; dirty = false; } finally { statsReadWriteLock.unlock(); } } return snapshot; }
throw new RestHandlerException("Checkpointing has not been enabled.", HttpResponseStatus.NOT_FOUND); } else { final CheckpointStatsCounts checkpointStatsCounts = checkpointStatsSnapshot.getCounts(); checkpointStatsCounts.getNumberOfFailedCheckpoints()); final CompletedCheckpointStatsSummary checkpointStatsSummary = checkpointStatsSnapshot.getSummaryStats(); final MinMaxAvgStats stateSize = checkpointStatsSummary.getStateSizeStats(); final MinMaxAvgStats duration = checkpointStatsSummary.getEndToEndDurationStats(); alignment.getAverage())); final CheckpointStatsHistory checkpointStatsHistory = checkpointStatsSnapshot.getHistory(); null; final RestoredCheckpointStats restoredCheckpointStats = checkpointStatsSnapshot.getLatestRestoredCheckpoint(); for (AbstractCheckpointStats abstractCheckpointStats : checkpointStatsSnapshot.getHistory().getCheckpoints()) { history.add(CheckpointStatistics.generateCheckpointStatistics(abstractCheckpointStats, false));
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size()); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { ResponseBody json = CheckpointStatistics.generateCheckpointStatistics(checkpoint, true); String path = getMessageHeaders().getTargetRestEndpointURL() .replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString()) .replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId())); archive.add(new ArchivedJson(path, json)); } return archive; } }
latestSnapshot = new CheckpointStatsSnapshot( counts.createSnapshot(), summary.createSnapshot(),
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { for (TaskStateStats subtaskStats : checkpoint.getAllTaskStateStats()) { String json = createSubtaskCheckpointDetailsJson(checkpoint, subtaskStats); String path = CHECKPOINT_STATS_DETAILS_SUBTASKS_REST_PATH .replace(":jobid", graph.getJobID().toString()) .replace(":checkpointid", String.valueOf(checkpoint.getCheckpointId())) .replace(":vertexid", subtaskStats.getJobVertexId().toString()); archive.add(new ArchivedJson(path, json)); } } return archive; } }
latestSnapshot = new CheckpointStatsSnapshot( counts.createSnapshot(), summary.createSnapshot(),
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size()); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { for (TaskStateStats subtaskStats : checkpoint.getAllTaskStateStats()) { ResponseBody json = createCheckpointDetails(checkpoint, subtaskStats); String path = getMessageHeaders().getTargetRestEndpointURL() .replace(':' + JobVertexIdPathParameter.KEY, graph.getJobID().toString()) .replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId())) .replace(':' + JobVertexIdPathParameter.KEY, subtaskStats.getJobVertexId().toString()); archive.add(new ArchivedJson(path, json)); } } return archive; }
latestSnapshot = new CheckpointStatsSnapshot( counts.createSnapshot(), summary.createSnapshot(),