@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { String json = createCheckpointDetailsJson(checkpoint); String path = CHECKPOINT_STATS_DETAILS_REST_PATH .replace(":jobid", graph.getJobID().toString()) .replace(":checkpointid", String.valueOf(checkpoint.getCheckpointId())); archive.add(new ArchivedJson(path, json)); } return archive; } }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { String json = createCheckpointDetailsJson(checkpoint); String path = CHECKPOINT_STATS_DETAILS_REST_PATH .replace(":jobid", graph.getJobID().toString()) .replace(":checkpointid", String.valueOf(checkpoint.getCheckpointId())); archive.add(new ArchivedJson(path, json)); } return archive; } }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size()); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { ResponseBody json = CheckpointStatistics.generateCheckpointStatistics(checkpoint, true); String path = getMessageHeaders().getTargetRestEndpointURL() .replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString()) .replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId())); archive.add(new ArchivedJson(path, json)); } return archive; } }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size()); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { ResponseBody json = CheckpointStatistics.generateCheckpointStatistics(checkpoint, true); String path = getMessageHeaders().getTargetRestEndpointURL() .replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString()) .replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId())); archive.add(new ArchivedJson(path, json)); } return archive; } }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size()); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { ResponseBody json = CheckpointStatistics.generateCheckpointStatistics(checkpoint, true); String path = getMessageHeaders().getTargetRestEndpointURL() .replace(':' + JobIDPathParameter.KEY, graph.getJobID().toString()) .replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId())); archive.add(new ArchivedJson(path, json)); } return archive; } }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { for (TaskStateStats subtaskStats : checkpoint.getAllTaskStateStats()) { String json = createSubtaskCheckpointDetailsJson(checkpoint, subtaskStats); String path = CHECKPOINT_STATS_DETAILS_SUBTASKS_REST_PATH .replace(":jobid", graph.getJobID().toString()) .replace(":checkpointid", String.valueOf(checkpoint.getCheckpointId())) .replace(":vertexid", subtaskStats.getJobVertexId().toString()); archive.add(new ArchivedJson(path, json)); } } return archive; } }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { for (TaskStateStats subtaskStats : checkpoint.getAllTaskStateStats()) { String json = createSubtaskCheckpointDetailsJson(checkpoint, subtaskStats); String path = CHECKPOINT_STATS_DETAILS_SUBTASKS_REST_PATH .replace(":jobid", graph.getJobID().toString()) .replace(":checkpointid", String.valueOf(checkpoint.getCheckpointId())) .replace(":vertexid", subtaskStats.getJobVertexId().toString()); archive.add(new ArchivedJson(path, json)); } } return archive; } }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size()); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { for (TaskStateStats subtaskStats : checkpoint.getAllTaskStateStats()) { ResponseBody json = createCheckpointDetails(checkpoint, subtaskStats); String path = getMessageHeaders().getTargetRestEndpointURL() .replace(':' + JobVertexIdPathParameter.KEY, graph.getJobID().toString()) .replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId())) .replace(':' + JobVertexIdPathParameter.KEY, subtaskStats.getJobVertexId().toString()); archive.add(new ArchivedJson(path, json)); } } return archive; }
private static String createCheckpointStatsJson(AccessExecutionGraph graph) throws IOException { StringWriter writer = new StringWriter(); JsonGenerator gen = JsonFactory.jacksonFactory.createGenerator(writer); CheckpointStatsSnapshot snapshot = graph.getCheckpointStatsSnapshot(); if (snapshot == null) { return "{}"; } gen.writeStartObject(); // Counts writeCounts(gen, snapshot.getCounts()); // Summary writeSummary(gen, snapshot.getSummaryStats()); CheckpointStatsHistory history = snapshot.getHistory(); // Latest writeLatestCheckpoints( gen, history.getLatestCompletedCheckpoint(), history.getLatestSavepoint(), history.getLatestFailedCheckpoint(), snapshot.getLatestRestoredCheckpoint()); // History writeHistory(gen, snapshot.getHistory()); gen.writeEndObject(); gen.close(); return writer.toString(); }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size()); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { for (TaskStateStats subtaskStats : checkpoint.getAllTaskStateStats()) { ResponseBody json = createCheckpointDetails(checkpoint, subtaskStats); String path = getMessageHeaders().getTargetRestEndpointURL() .replace(':' + JobVertexIdPathParameter.KEY, graph.getJobID().toString()) .replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId())) .replace(':' + JobVertexIdPathParameter.KEY, subtaskStats.getJobVertexId().toString()); archive.add(new ArchivedJson(path, json)); } } return archive; }
@Override public Collection<ArchivedJson> archiveJsonWithPath(AccessExecutionGraph graph) throws IOException { CheckpointStatsSnapshot stats = graph.getCheckpointStatsSnapshot(); if (stats == null) { return Collections.emptyList(); } CheckpointStatsHistory history = stats.getHistory(); List<ArchivedJson> archive = new ArrayList<>(history.getCheckpoints().size()); for (AbstractCheckpointStats checkpoint : history.getCheckpoints()) { for (TaskStateStats subtaskStats : checkpoint.getAllTaskStateStats()) { ResponseBody json = createCheckpointDetails(checkpoint, subtaskStats); String path = getMessageHeaders().getTargetRestEndpointURL() .replace(':' + JobVertexIdPathParameter.KEY, graph.getJobID().toString()) .replace(':' + CheckpointIdPathParameter.KEY, String.valueOf(checkpoint.getCheckpointId())) .replace(':' + JobVertexIdPathParameter.KEY, subtaskStats.getJobVertexId().toString()); archive.add(new ArchivedJson(path, json)); } } return archive; }
AbstractCheckpointStats checkpoint = snapshot.getHistory().getCheckpointById(checkpointId);
@Override public String handleRequest(AccessExecutionGraph graph, Map<String, String> params) throws Exception { long checkpointId = parseCheckpointId(params); if (checkpointId == -1) { return "{}"; } CheckpointStatsSnapshot snapshot = graph.getCheckpointStatsSnapshot(); if (snapshot == null) { return "{}"; } AbstractCheckpointStats checkpoint = snapshot.getHistory().getCheckpointById(checkpointId); if (checkpoint != null) { cache.tryAdd(checkpoint); } else { checkpoint = cache.tryGet(checkpointId); if (checkpoint == null) { return "{}"; } } return createCheckpointDetailsJson(checkpoint); }
private static String createCheckpointStatsJson(AccessExecutionGraph graph) throws IOException { StringWriter writer = new StringWriter(); JsonGenerator gen = JsonFactory.JACKSON_FACTORY.createGenerator(writer); CheckpointStatsSnapshot snapshot = graph.getCheckpointStatsSnapshot(); if (snapshot == null) { return "{}"; } gen.writeStartObject(); // Counts writeCounts(gen, snapshot.getCounts()); // Summary writeSummary(gen, snapshot.getSummaryStats()); CheckpointStatsHistory history = snapshot.getHistory(); // Latest writeLatestCheckpoints( gen, history.getLatestCompletedCheckpoint(), history.getLatestSavepoint(), history.getLatestFailedCheckpoint(), snapshot.getLatestRestoredCheckpoint()); // History writeHistory(gen, snapshot.getHistory()); gen.writeEndObject(); gen.close(); return writer.toString(); }
@Override public CompletableFuture<String> handleRequest(AccessExecutionGraph graph, Map<String, String> params) { return CompletableFuture.supplyAsync( () -> { long checkpointId = parseCheckpointId(params); if (checkpointId == -1) { return "{}"; } CheckpointStatsSnapshot snapshot = graph.getCheckpointStatsSnapshot(); if (snapshot == null) { return "{}"; } AbstractCheckpointStats checkpoint = snapshot.getHistory().getCheckpointById(checkpointId); if (checkpoint != null) { cache.tryAdd(checkpoint); } else { checkpoint = cache.tryGet(checkpointId); if (checkpoint == null) { return "{}"; } } try { return createCheckpointDetailsJson(checkpoint); } catch (IOException e) { throw new CompletionException(new FlinkException("Could not create checkpoint details json.", e)); } }, executor); }
@Override protected R handleRequest(HandlerRequest<EmptyRequestBody, M> request, AccessExecutionGraph executionGraph) throws RestHandlerException { final long checkpointId = request.getPathParameter(CheckpointIdPathParameter.class); final CheckpointStatsSnapshot checkpointStatsSnapshot = executionGraph.getCheckpointStatsSnapshot(); if (checkpointStatsSnapshot != null) { AbstractCheckpointStats checkpointStats = checkpointStatsSnapshot.getHistory().getCheckpointById(checkpointId); if (checkpointStats != null) { checkpointStatsCache.tryAdd(checkpointStats); } else { checkpointStats = checkpointStatsCache.tryGet(checkpointId); } if (checkpointStats != null) { return handleCheckpointRequest(request, checkpointStats); } else { throw new RestHandlerException("Could not find checkpointing statistics for checkpoint " + checkpointId + '.', HttpResponseStatus.NOT_FOUND); } } else { throw new RestHandlerException("Checkpointing was not enabled for job " + executionGraph.getJobID() + '.', HttpResponseStatus.NOT_FOUND); } }
@Override protected R handleRequest(HandlerRequest<EmptyRequestBody, M> request, AccessExecutionGraph executionGraph) throws RestHandlerException { final long checkpointId = request.getPathParameter(CheckpointIdPathParameter.class); final CheckpointStatsSnapshot checkpointStatsSnapshot = executionGraph.getCheckpointStatsSnapshot(); if (checkpointStatsSnapshot != null) { AbstractCheckpointStats checkpointStats = checkpointStatsSnapshot.getHistory().getCheckpointById(checkpointId); if (checkpointStats != null) { checkpointStatsCache.tryAdd(checkpointStats); } else { checkpointStats = checkpointStatsCache.tryGet(checkpointId); } if (checkpointStats != null) { return handleCheckpointRequest(request, checkpointStats); } else { throw new RestHandlerException("Could not find checkpointing statistics for checkpoint " + checkpointId + '.', HttpResponseStatus.NOT_FOUND); } } else { throw new RestHandlerException("Checkpointing was not enabled for job " + executionGraph.getJobID() + '.', HttpResponseStatus.NOT_FOUND); } }
@Override protected R handleRequest(HandlerRequest<EmptyRequestBody, M> request, AccessExecutionGraph executionGraph) throws RestHandlerException { final long checkpointId = request.getPathParameter(CheckpointIdPathParameter.class); final CheckpointStatsSnapshot checkpointStatsSnapshot = executionGraph.getCheckpointStatsSnapshot(); if (checkpointStatsSnapshot != null) { AbstractCheckpointStats checkpointStats = checkpointStatsSnapshot.getHistory().getCheckpointById(checkpointId); if (checkpointStats != null) { checkpointStatsCache.tryAdd(checkpointStats); } else { checkpointStats = checkpointStatsCache.tryGet(checkpointId); } if (checkpointStats != null) { return handleCheckpointRequest(request, checkpointStats); } else { throw new RestHandlerException("Could not find checkpointing statistics for checkpoint " + checkpointId + '.', HttpResponseStatus.NOT_FOUND); } } else { throw new RestHandlerException("Checkpointing was not enabled for job " + executionGraph.getJobID() + '.', HttpResponseStatus.NOT_FOUND); } }
@Override public String handleRequest(AccessExecutionGraph graph, Map<String, String> params) throws Exception { long checkpointId = CheckpointStatsDetailsHandler.parseCheckpointId(params); if (checkpointId == -1) { return "{}"; } JobVertexID vertexId = AbstractJobVertexRequestHandler.parseJobVertexId(params); if (vertexId == null) { return "{}"; } CheckpointStatsSnapshot snapshot = graph.getCheckpointStatsSnapshot(); if (snapshot == null) { return "{}"; } AbstractCheckpointStats checkpoint = snapshot.getHistory().getCheckpointById(checkpointId); if (checkpoint != null) { cache.tryAdd(checkpoint); } else { checkpoint = cache.tryGet(checkpointId); if (checkpoint == null) { return "{}"; } } TaskStateStats taskStats = checkpoint.getTaskStateStats(vertexId); if (taskStats == null) { return "{}"; } return createSubtaskCheckpointDetailsJson(checkpoint, taskStats); }
alignment.getAverage())); final CheckpointStatsHistory checkpointStatsHistory = checkpointStatsSnapshot.getHistory(); for (AbstractCheckpointStats abstractCheckpointStats : checkpointStatsSnapshot.getHistory().getCheckpoints()) { history.add(CheckpointStatistics.generateCheckpointStatistics(abstractCheckpointStats, false));