env.getTaskStateManager().createLocalRecoveryConfig();
@Override public void acknowledgeCheckpoint(long checkpointId, CheckpointMetrics checkpointMetrics, TaskStateSnapshot subtaskState) { taskStateManager.reportTaskStateSnapshots( new CheckpointMetaData(checkpointId, 0L), checkpointMetrics, subtaskState, null); }
taskStateManager.prioritizedOperatorState(operatorID);
@Override public void run() { try { invokable.notifyCheckpointComplete(checkpointID); taskStateManager.notifyCheckpointComplete(checkpointID); } catch (Throwable t) { if (getExecutionState() == ExecutionState.RUNNING) { // fail task if checkpoint confirmation failed. failExternally(new RuntimeException( "Error while confirming checkpoint", t)); } } } };
@Override public void run() { try { invokable.notifyCheckpointComplete(checkpointID); taskStateManager.notifyCheckpointComplete(checkpointID); } catch (Throwable t) { if (getExecutionState() == ExecutionState.RUNNING) { // fail task if checkpoint confirmation failed. failExternally(new RuntimeException( "Error while confirming checkpoint", t)); } } } };
@Override public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend( Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry) { TaskStateManager taskStateManager = env.getTaskStateManager(); return new HeapKeyedStateBackend<>( kvStateRegistry, keySerializer, env.getUserClassLoader(), numberOfKeyGroups, keyGroupRange, isUsingAsynchronousSnapshots(), env.getExecutionConfig(), taskStateManager.createLocalRecoveryConfig()); }
private void reportCompletedSnapshotStates( TaskStateSnapshot acknowledgedTaskStateSnapshot, TaskStateSnapshot localTaskStateSnapshot, long asyncDurationMillis) { TaskStateManager taskStateManager = owner.getEnvironment().getTaskStateManager(); boolean hasAckState = acknowledgedTaskStateSnapshot.hasState(); boolean hasLocalState = localTaskStateSnapshot.hasState(); Preconditions.checkState(hasAckState || !hasLocalState, "Found cached state but no corresponding primary state is reported to the job " + "manager. This indicates a problem."); // we signal stateless tasks by reporting null, so that there are no attempts to assign empty state // to stateless tasks on restore. This enables simple job modifications that only concern // stateless without the need to assign them uids to match their (always empty) states. taskStateManager.reportTaskStateSnapshots( checkpointMetaData, checkpointMetrics, hasAckState ? acknowledgedTaskStateSnapshot : null, hasLocalState ? localTaskStateSnapshot : null); LOG.debug("{} - finished asynchronous part of checkpoint {}. Asynchronous duration: {} ms", owner.getName(), checkpointMetaData.getCheckpointId(), asyncDurationMillis); LOG.trace("{} - reported the following states in snapshot for checkpoint {}: {}.", owner.getName(), checkpointMetaData.getCheckpointId(), acknowledgedTaskStateSnapshot); }
@Override public void run() { try { invokable.notifyCheckpointComplete(checkpointID); taskStateManager.notifyCheckpointComplete(checkpointID); } catch (Throwable t) { if (getExecutionState() == ExecutionState.RUNNING) { // fail task if checkpoint confirmation failed. failExternally(new RuntimeException( "Error while confirming checkpoint", t)); } } } };
taskStateManager.prioritizedOperatorState(operatorID);
@Override public AbstractInternalStateBackend createInternalStateBackend( Environment env, String operatorIdentifier, int numberOfGroups, KeyGroupRange keyGroupRange) { return new HeapInternalStateBackend( numberOfGroups, keyGroupRange, env.getUserClassLoader(), env.getTaskStateManager().createLocalRecoveryConfig(), env.getTaskKvStateRegistry(), isUsingAsynchronousSnapshots(), env.getExecutionConfig() ); }
private void reportCompletedSnapshotStates( TaskStateSnapshot acknowledgedTaskStateSnapshot, TaskStateSnapshot localTaskStateSnapshot, long asyncDurationMillis) { TaskStateManager taskStateManager = owner.getEnvironment().getTaskStateManager(); boolean hasAckState = acknowledgedTaskStateSnapshot.hasState(); boolean hasLocalState = localTaskStateSnapshot.hasState(); Preconditions.checkState(hasAckState || !hasLocalState, "Found cached state but no corresponding primary state is reported to the job " + "manager. This indicates a problem."); // we signal stateless tasks by reporting null, so that there are no attempts to assign empty state // to stateless tasks on restore. This enables simple job modifications that only concern // stateless without the need to assign them uids to match their (always empty) states. taskStateManager.reportTaskStateSnapshots( checkpointMetaData, checkpointMetrics, hasAckState ? acknowledgedTaskStateSnapshot : null, hasLocalState ? localTaskStateSnapshot : null); LOG.debug("{} - finished asynchronous part of checkpoint {}. Asynchronous duration: {} ms", owner.getName(), checkpointMetaData.getCheckpointId(), asyncDurationMillis); LOG.trace("{} - reported the following states in snapshot for checkpoint {}: {}.", owner.getName(), checkpointMetaData.getCheckpointId(), acknowledgedTaskStateSnapshot); }
taskStateManager.prioritizedOperatorState(operatorID);
@Override public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend( Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry) { TaskStateManager taskStateManager = env.getTaskStateManager(); LocalRecoveryConfig localRecoveryConfig = taskStateManager.createLocalRecoveryConfig(); return new HeapKeyedStateBackend<>( kvStateRegistry, keySerializer, env.getUserClassLoader(), numberOfKeyGroups, keyGroupRange, isUsingAsynchronousSnapshots(), env.getExecutionConfig(), localRecoveryConfig); }
private void reportCompletedSnapshotStates( TaskStateSnapshot acknowledgedTaskStateSnapshot, TaskStateSnapshot localTaskStateSnapshot, long asyncDurationMillis) { TaskStateManager taskStateManager = owner.getEnvironment().getTaskStateManager(); boolean hasAckState = acknowledgedTaskStateSnapshot.hasState(); boolean hasLocalState = localTaskStateSnapshot.hasState(); Preconditions.checkState(hasAckState || !hasLocalState, "Found cached state but no corresponding primary state is reported to the job " + "manager. This indicates a problem."); // we signal stateless tasks by reporting null, so that there are no attempts to assign empty state // to stateless tasks on restore. This enables simple job modifications that only concern // stateless without the need to assign them uids to match their (always empty) states. taskStateManager.reportTaskStateSnapshots( checkpointMetaData, checkpointMetrics, hasAckState ? acknowledgedTaskStateSnapshot : null, hasLocalState ? localTaskStateSnapshot : null); LOG.debug("{} - finished asynchronous part of checkpoint {}. Asynchronous duration: {} ms", owner.getName(), checkpointMetaData.getCheckpointId(), asyncDurationMillis); LOG.trace("{} - reported the following states in snapshot for checkpoint {}: {}.", owner.getName(), checkpointMetaData.getCheckpointId(), acknowledgedTaskStateSnapshot); }
@Override public AbstractInternalStateBackend createInternalStateBackend( Environment env, String operatorIdentifier, int numberOfGroups, KeyGroupRange keyGroupRange) { return new HeapInternalStateBackend( numberOfGroups, keyGroupRange, env.getUserClassLoader(), env.getTaskStateManager().createLocalRecoveryConfig(), env.getTaskKvStateRegistry(), isUsingAsynchronousSnapshots(), env.getExecutionConfig() ); }
@Override public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend( Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry, TtlTimeProvider ttlTimeProvider, MetricGroup metricGroup) { TaskStateManager taskStateManager = env.getTaskStateManager(); LocalRecoveryConfig localRecoveryConfig = taskStateManager.createLocalRecoveryConfig(); HeapPriorityQueueSetFactory priorityQueueSetFactory = new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128); return new HeapKeyedStateBackend<>( kvStateRegistry, keySerializer, env.getUserClassLoader(), numberOfKeyGroups, keyGroupRange, isUsingAsynchronousSnapshots(), env.getExecutionConfig(), localRecoveryConfig, priorityQueueSetFactory, ttlTimeProvider); }
@Override public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend( Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry, TtlTimeProvider ttlTimeProvider, MetricGroup metricGroup) { TaskStateManager taskStateManager = env.getTaskStateManager(); HeapPriorityQueueSetFactory priorityQueueSetFactory = new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128); return new HeapKeyedStateBackend<>( kvStateRegistry, keySerializer, env.getUserClassLoader(), numberOfKeyGroups, keyGroupRange, isUsingAsynchronousSnapshots(), env.getExecutionConfig(), taskStateManager.createLocalRecoveryConfig(), priorityQueueSetFactory, ttlTimeProvider); }
@Override public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend( Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry, TtlTimeProvider ttlTimeProvider, MetricGroup metricGroup) { TaskStateManager taskStateManager = env.getTaskStateManager(); HeapPriorityQueueSetFactory priorityQueueSetFactory = new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128); return new HeapKeyedStateBackend<>( kvStateRegistry, keySerializer, env.getUserClassLoader(), numberOfKeyGroups, keyGroupRange, isUsingAsynchronousSnapshots(), env.getExecutionConfig(), taskStateManager.createLocalRecoveryConfig(), priorityQueueSetFactory, ttlTimeProvider); }
@Override public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend( Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry, TtlTimeProvider ttlTimeProvider, MetricGroup metricGroup) { TaskStateManager taskStateManager = env.getTaskStateManager(); LocalRecoveryConfig localRecoveryConfig = taskStateManager.createLocalRecoveryConfig(); HeapPriorityQueueSetFactory priorityQueueSetFactory = new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128); return new HeapKeyedStateBackend<>( kvStateRegistry, keySerializer, env.getUserClassLoader(), numberOfKeyGroups, keyGroupRange, isUsingAsynchronousSnapshots(), env.getExecutionConfig(), localRecoveryConfig, priorityQueueSetFactory, ttlTimeProvider); }
env.getTaskStateManager().createLocalRecoveryConfig();