private static void writeCounts(JsonGenerator gen, CheckpointStatsCounts counts) throws IOException { gen.writeObjectFieldStart("counts"); gen.writeNumberField("restored", counts.getNumberOfRestoredCheckpoints()); gen.writeNumberField("total", counts.getTotalNumberOfCheckpoints()); gen.writeNumberField("in_progress", counts.getNumberOfInProgressCheckpoints()); gen.writeNumberField("completed", counts.getNumberOfCompletedCheckpoints()); gen.writeNumberField("failed", counts.getNumberOfFailedCheckpoints()); gen.writeEndObject(); }
/** * Creates a snapshot of the current state. * * @return Snapshot of the current state. */ CheckpointStatsCounts createSnapshot() { return new CheckpointStatsCounts( numRestoredCheckpoints, numTotalCheckpoints, numInProgressCheckpoints, numCompletedCheckpoints, numFailedCheckpoints); }
/** * Creates a new snapshot of the available stats. * * @return The latest statistics snapshot. */ public CheckpointStatsSnapshot createSnapshot() { CheckpointStatsSnapshot snapshot = latestSnapshot; // Only create a new snapshot if dirty and no update in progress, // because we don't want to block the coordinator. if (dirty && statsReadWriteLock.tryLock()) { try { // Create a new snapshot snapshot = new CheckpointStatsSnapshot( counts.createSnapshot(), summary.createSnapshot(), history.createSnapshot(), latestRestoredCheckpoint); latestSnapshot = snapshot; dirty = false; } finally { statsReadWriteLock.unlock(); } } return snapshot; }
@Override public Long getValue() { return counts.getTotalNumberOfCheckpoints(); } }
@Override public Integer getValue() { return counts.getNumberOfInProgressCheckpoints(); } }
@Override public Long getValue() { return counts.getNumberOfFailedCheckpoints(); } }
@Override public Long getValue() { return counts.getNumberOfCompletedCheckpoints(); } }
/** * Callback when a checkpoint fails. * * @param failed The failed checkpoint stats. */ private void reportFailedCheckpoint(FailedCheckpointStats failed) { statsReadWriteLock.lock(); try { counts.incrementFailedCheckpoints(); history.replacePendingCheckpointById(failed); dirty = true; } finally { statsReadWriteLock.unlock(); } }
/** * Callback when a checkpoint completes. * * @param completed The completed checkpoint stats. */ private void reportCompletedCheckpoint(CompletedCheckpointStats completed) { statsReadWriteLock.lock(); try { latestCompletedCheckpoint = completed; counts.incrementCompletedCheckpoints(); history.replacePendingCheckpointById(completed); summary.updateSummary(completed); dirty = true; } finally { statsReadWriteLock.unlock(); } }
/** * Creates a new pending checkpoint tracker. * * @param checkpointId ID of the checkpoint. * @param triggerTimestamp Trigger timestamp of the checkpoint. * @param props The checkpoint properties. * @return Tracker for statistics gathering. */ PendingCheckpointStats reportPendingCheckpoint( long checkpointId, long triggerTimestamp, CheckpointProperties props) { ConcurrentHashMap<JobVertexID, TaskStateStats> taskStateStats = createEmptyTaskStateStatsMap(); PendingCheckpointStats pending = new PendingCheckpointStats( checkpointId, triggerTimestamp, props, totalSubtaskCount, taskStateStats, new PendingCheckpointStatsCallback()); statsReadWriteLock.lock(); try { counts.incrementInProgressCheckpoints(); history.addInProgressCheckpoint(pending); dirty = true; } finally { statsReadWriteLock.unlock(); } return pending; }
/** * Creates a new snapshot of the available stats. * * @return The latest statistics snapshot. */ public CheckpointStatsSnapshot createSnapshot() { CheckpointStatsSnapshot snapshot = latestSnapshot; // Only create a new snapshot if dirty and no update in progress, // because we don't want to block the coordinator. if (dirty && statsReadWriteLock.tryLock()) { try { // Create a new snapshot snapshot = new CheckpointStatsSnapshot( counts.createSnapshot(), summary.createSnapshot(), history.createSnapshot(), latestRestoredCheckpoint); latestSnapshot = snapshot; dirty = false; } finally { statsReadWriteLock.unlock(); } } return snapshot; }
@Override public Long getValue() { return counts.getTotalNumberOfCheckpoints(); } }
@Override public Integer getValue() { return counts.getNumberOfInProgressCheckpoints(); } }
@Override public Long getValue() { return counts.getNumberOfFailedCheckpoints(); } }
@Override public Long getValue() { return counts.getNumberOfCompletedCheckpoints(); } }
/** * Callback when a checkpoint fails. * * @param failed The failed checkpoint stats. */ private void reportFailedCheckpoint(FailedCheckpointStats failed) { statsReadWriteLock.lock(); try { counts.incrementFailedCheckpoints(); history.replacePendingCheckpointById(failed); dirty = true; } finally { statsReadWriteLock.unlock(); } }
/** * Callback when a checkpoint completes. * * @param completed The completed checkpoint stats. */ private void reportCompletedCheckpoint(CompletedCheckpointStats completed) { statsReadWriteLock.lock(); try { latestCompletedCheckpoint = completed; counts.incrementCompletedCheckpoints(); history.replacePendingCheckpointById(completed); summary.updateSummary(completed); dirty = true; } finally { statsReadWriteLock.unlock(); } }
/** * Creates a new pending checkpoint tracker. * * @param checkpointId ID of the checkpoint. * @param triggerTimestamp Trigger timestamp of the checkpoint. * @param props The checkpoint properties. * @return Tracker for statistics gathering. */ PendingCheckpointStats reportPendingCheckpoint( long checkpointId, long triggerTimestamp, CheckpointProperties props) { ConcurrentHashMap<JobVertexID, TaskStateStats> taskStateStats = createEmptyTaskStateStatsMap(); PendingCheckpointStats pending = new PendingCheckpointStats( checkpointId, triggerTimestamp, props, totalSubtaskCount, taskStateStats, new PendingCheckpointStatsCallback()); statsReadWriteLock.lock(); try { counts.incrementInProgressCheckpoints(); history.addInProgressCheckpoint(pending); dirty = true; } finally { statsReadWriteLock.unlock(); } return pending; }
private static void writeCounts(JsonGenerator gen, CheckpointStatsCounts counts) throws IOException { gen.writeObjectFieldStart(CheckpointingStatistics.FIELD_NAME_COUNTS); gen.writeNumberField(CheckpointingStatistics.Counts.FIELD_NAME_RESTORED_CHECKPOINTS, counts.getNumberOfRestoredCheckpoints()); gen.writeNumberField(CheckpointingStatistics.Counts.FIELD_NAME_TOTAL_CHECKPOINTS, counts.getTotalNumberOfCheckpoints()); gen.writeNumberField(CheckpointingStatistics.Counts.FIELD_NAME_IN_PROGRESS_CHECKPOINTS, counts.getNumberOfInProgressCheckpoints()); gen.writeNumberField(CheckpointingStatistics.Counts.FIELD_NAME_COMPLETED_CHECKPOINTS, counts.getNumberOfCompletedCheckpoints()); gen.writeNumberField(CheckpointingStatistics.Counts.FIELD_NAME_FAILED_CHECKPOINTS, counts.getNumberOfFailedCheckpoints()); gen.writeEndObject(); }
/** * Creates a new snapshot of the available stats. * * @return The latest statistics snapshot. */ public CheckpointStatsSnapshot createSnapshot() { CheckpointStatsSnapshot snapshot = latestSnapshot; // Only create a new snapshot if dirty and no update in progress, // because we don't want to block the coordinator. if (dirty && statsReadWriteLock.tryLock()) { try { // Create a new snapshot snapshot = new CheckpointStatsSnapshot( counts.createSnapshot(), summary.createSnapshot(), history.createSnapshot(), latestRestoredCheckpoint); latestSnapshot = snapshot; dirty = false; } finally { statsReadWriteLock.unlock(); } } return snapshot; }