/** * Creates a new {@code RocksDBStateBackend} that stores its checkpoint data in the * file system and location defined by the given URI. * * <p>A state backend that stores checkpoints in HDFS or S3 must specify the file system * host and port in the URI, or have the Hadoop configuration that describes the file system * (host / high-availability group / possibly credentials) either referenced from the Flink * config, or included in the classpath. * * @param checkpointDataUri The URI describing the filesystem and path to the checkpoint data directory. * @throws IOException Thrown, if no file system can be found for the scheme in the URI. */ @SuppressWarnings("deprecation") public RocksDBStateBackend(URI checkpointDataUri) throws IOException { this(new FsStateBackend(checkpointDataUri)); }
@Override public FsStateBackend createFromConfig(Configuration config) throws IllegalConfigurationException { // we need to explicitly read the checkpoint directory here, because that // is a required constructor parameter final String checkpointDir = config.getString(CheckpointingOptions.CHECKPOINTS_DIRECTORY); if (checkpointDir == null) { throw new IllegalConfigurationException( "Cannot create the file system state backend: The configuration does not specify the " + "checkpoint directory '" + CheckpointingOptions.CHECKPOINTS_DIRECTORY.key() + '\''); } try { return new FsStateBackend(checkpointDir).configure(config); } catch (IllegalArgumentException e) { throw new IllegalConfigurationException("Invalid configuration for the state backend", e); } } }
@Override public CheckpointStorage createCheckpointStorage(JobID jobId) throws IOException { checkNotNull(jobId, "jobId"); return new FsCheckpointStorage(getCheckpointPath(), getSavepointPath(), jobId, getMinFileSizeThreshold()); }
@Override public String toString() { return "File State Backend (" + "checkpoints: '" + getCheckpointPath() + "', savepoints: '" + getSavepointPath() + "', asynchronous: " + asynchronousSnapshots + ", fileStateThreshold: " + fileStateThreshold + ")"; } }
@Override public OperatorStateBackend createOperatorStateBackend( Environment env, String operatorIdentifier) { return new DefaultOperatorStateBackend( env.getUserClassLoader(), env.getExecutionConfig(), isUsingAsynchronousSnapshots()); }
/** * Gets the base directory where all the checkpoints are stored. * The job-specific checkpoint directory is created inside this directory. * * @return The base directory for checkpoints. * * @deprecated Deprecated in favor of {@link #getCheckpointPath()}. */ @Deprecated public Path getBasePath() { return getCheckpointPath(); }
@Override public FsStateBackend createFromConfig(Configuration config) throws IllegalConfigurationException { // we need to explicitly read the checkpoint directory here, because that // is a required constructor parameter final String checkpointDir = config.getString(CheckpointingOptions.CHECKPOINTS_DIRECTORY); if (checkpointDir == null) { throw new IllegalConfigurationException( "Cannot create the file system state backend: The configuration does not specify the " + "checkpoint directory '" + CheckpointingOptions.CHECKPOINTS_DIRECTORY.key() + '\''); } try { return new FsStateBackend(checkpointDir).configure(config); } catch (IllegalArgumentException e) { throw new IllegalConfigurationException("Invalid configuration for the state backend", e); } } }
@Override public CheckpointStorage createCheckpointStorage(JobID jobId) throws IOException { checkNotNull(jobId, "jobId"); return new FsCheckpointStorage(getCheckpointPath(), getSavepointPath(), jobId, getMinFileSizeThreshold()); }
@Override public String toString() { return "File State Backend (" + "checkpoints: '" + getCheckpointPath() + "', savepoints: '" + getSavepointPath() + "', asynchronous: " + asynchronousSnapshots + ", fileStateThreshold: " + fileStateThreshold + ")"; } }
@Override public OperatorStateBackend createOperatorStateBackend( Environment env, String operatorIdentifier) { return new DefaultOperatorStateBackend( env.getUserClassLoader(), env.getExecutionConfig(), isUsingAsynchronousSnapshots()); }
/** * Gets the base directory where all the checkpoints are stored. * The job-specific checkpoint directory is created inside this directory. * * @return The base directory for checkpoints. * * @deprecated Deprecated in favor of {@link #getCheckpointPath()}. */ @Deprecated public Path getBasePath() { return getCheckpointPath(); }
/** * Creates a new {@code RocksDBStateBackend} that stores its checkpoint data in the * file system and location defined by the given URI. * * <p>A state backend that stores checkpoints in HDFS or S3 must specify the file system * host and port in the URI, or have the Hadoop configuration that describes the file system * (host / high-availability group / possibly credentials) either referenced from the Flink * config, or included in the classpath. * * @param checkpointDataUri The URI describing the filesystem and path to the checkpoint data directory. * @param enableIncrementalCheckpointing True if incremental checkpointing is enabled. * @throws IOException Thrown, if no file system can be found for the scheme in the URI. */ @SuppressWarnings("deprecation") public RocksDBStateBackend(URI checkpointDataUri, boolean enableIncrementalCheckpointing) throws IOException { this(new FsStateBackend(checkpointDataUri), enableIncrementalCheckpointing); }
@Override public FsStateBackend createFromConfig(Configuration config) throws IllegalConfigurationException { // we need to explicitly read the checkpoint directory here, because that // is a required constructor parameter final String checkpointDir = config.getString(CheckpointingOptions.CHECKPOINTS_DIRECTORY); if (checkpointDir == null) { throw new IllegalConfigurationException( "Cannot create the file system state backend: The configuration does not specify the " + "checkpoint directory '" + CheckpointingOptions.CHECKPOINTS_DIRECTORY.key() + '\''); } try { return new FsStateBackend(checkpointDir).configure(config); } catch (IllegalArgumentException e) { throw new IllegalConfigurationException("Invalid configuration for the state backend", e); } } }
@Override public CheckpointStorage createCheckpointStorage(JobID jobId) throws IOException { checkNotNull(jobId, "jobId"); return new FsCheckpointStorage( getCheckpointPath(), getSavepointPath(), jobId, createCheckpointSubDirs.getOrDefault(CheckpointingOptions.CHCKPOINTS_CREATE_SUBDIRS.defaultValue()), getMinFileSizeThreshold()); }
@Override public String toString() { return "File State Backend (" + "checkpoints: '" + getCheckpointPath() + "', savepoints: '" + getSavepointPath() + "', asynchronous: " + asynchronousSnapshots + ", fileStateThreshold: " + fileStateThreshold + ")"; } }
@Override public OperatorStateBackend createOperatorStateBackend( Environment env, String operatorIdentifier) { return new DefaultOperatorStateBackend( env.getUserClassLoader(), env.getExecutionConfig(), isUsingAsynchronousSnapshots()); }
/** * Gets the base directory where all the checkpoints are stored. * The job-specific checkpoint directory is created inside this directory. * * @return The base directory for checkpoints. * * @deprecated Deprecated in favor of {@link #getCheckpointPath()}. */ @Deprecated public Path getBasePath() { return getCheckpointPath(); }
private FsStateBackend createFsStateBackend(File checkpointDir) throws IOException { return new FsStateBackend(checkpointDir.toURI().toString(), true); }
/** * Private constructor that creates a re-configured copy of the state backend. * * @param original The state backend to re-configure * @param configuration The configuration */ private FsStateBackend(FsStateBackend original, Configuration configuration) { super(original.getCheckpointPath(), original.getSavepointPath(), configuration); // if asynchronous snapshots were configured, use that setting, // else check the configuration this.asynchronousSnapshots = original.asynchronousSnapshots.resolveUndefined( configuration.getBoolean(CheckpointingOptions.ASYNC_SNAPSHOTS)); final int sizeThreshold = original.fileStateThreshold >= 0 ? original.fileStateThreshold : configuration.getInteger(CheckpointingOptions.FS_SMALL_FILE_THRESHOLD); if (sizeThreshold >= 0 && sizeThreshold <= MAX_FILE_STATE_THRESHOLD) { this.fileStateThreshold = sizeThreshold; } else { this.fileStateThreshold = CheckpointingOptions.FS_SMALL_FILE_THRESHOLD.defaultValue(); // because this is the only place we (unlikely) ever log, we lazily // create the logger here LoggerFactory.getLogger(AbstractFileStateBackend.class).warn( "Ignoring invalid file size threshold value ({}): {} - using default value {} instead.", CheckpointingOptions.FS_SMALL_FILE_THRESHOLD.key(), sizeThreshold, CheckpointingOptions.FS_SMALL_FILE_THRESHOLD.defaultValue()); } }
@Override public <K> AbstractKeyedStateBackend<K> createKeyedStateBackend( Environment env, JobID jobID, String operatorIdentifier, TypeSerializer<K> keySerializer, int numberOfKeyGroups, KeyGroupRange keyGroupRange, TaskKvStateRegistry kvStateRegistry) { TaskStateManager taskStateManager = env.getTaskStateManager(); LocalRecoveryConfig localRecoveryConfig = taskStateManager.createLocalRecoveryConfig(); return new HeapKeyedStateBackend<>( kvStateRegistry, keySerializer, env.getUserClassLoader(), numberOfKeyGroups, keyGroupRange, isUsingAsynchronousSnapshots(), env.getExecutionConfig(), localRecoveryConfig); }