/** * Creates a new {@code RocksDBStateBackend} that stores its checkpoint data in the * file system and location defined by the given URI. * * <p>A state backend that stores checkpoints in HDFS or S3 must specify the file system * host and port in the URI, or have the Hadoop configuration that describes the file system * (host / high-availability group / possibly credentials) either referenced from the Flink * config, or included in the classpath. * * @param checkpointDataUri The URI describing the filesystem and path to the checkpoint data directory. * @throws IOException Thrown, if no file system can be found for the scheme in the URI. */ @SuppressWarnings("deprecation") public RocksDBStateBackend(URI checkpointDataUri) throws IOException { this(new FsStateBackend(checkpointDataUri)); }
@Override protected CheckpointStorageLocation createSavepointLocation(FileSystem fs, Path location) throws IOException { final CheckpointStorageLocationReference reference = encodePathAsReference(location); return new FsCheckpointStorageLocation(fs, location, location, location, reference, fileSizeThreshold); } }
/** * Creates a new {@code RocksDBStateBackend} that stores its checkpoint data in the * file system and location defined by the given URI. * * <p>A state backend that stores checkpoints in HDFS or S3 must specify the file system * host and port in the URI, or have the Hadoop configuration that describes the file system * (host / high-availability group / possibly credentials) either referenced from the Flink * config, or included in the classpath. * * @param checkpointDataUri The URI describing the filesystem and path to the checkpoint data directory. * @param enableIncrementalCheckpointing True if incremental checkpointing is enabled. * @throws IOException Thrown, if no file system can be found for the scheme in the URI. */ @SuppressWarnings("deprecation") public RocksDBStateBackend(URI checkpointDataUri, boolean enableIncrementalCheckpointing) throws IOException { this(new FsStateBackend(checkpointDataUri), enableIncrementalCheckpointing); }
private FsStateBackend createFsStateBackend(File checkpointDir) throws IOException { return new FsStateBackend(checkpointDir.toURI().toString(), true); }
protected final FsStateBackend createStateBackendInternal() throws IOException { File checkpointDir = temporaryFolder.newFolder(); return new FsStateBackend(checkpointDir.toURI()); }
@Override protected StateBackend createStateBackend() throws Exception { return new FsStateBackend(temporaryFolder.newFolder().toURI().toString()); }
@Override protected StateBackend createStateBackend() throws Exception { return new FsStateBackend(temporaryFolder.newFolder().toURI().toString()); }
@Test public void testCheckpointedStreamingProgramIncrementalRocksDB() throws Exception { testCheckpointedStreamingProgram( new RocksDBStateBackend( new FsStateBackend(temporaryFolder.newFolder().getAbsoluteFile().toURI(), 16), true)); }
@Test public void testWithFsBackendAsync() throws Exception { FsStateBackend asyncFsBackend = new FsStateBackend(tmpFolder.newFolder().toURI().toString(), true); testProgramWithBackend(asyncFsBackend); }
@Test public void testWithFsBackendSync() throws Exception { FsStateBackend syncFsBackend = new FsStateBackend(tmpFolder.newFolder().toURI().toString(), false); testProgramWithBackend(syncFsBackend); }
private void setupRocksDB(int fileSizeThreshold, boolean incrementalCheckpoints) throws IOException { String rocksDb = tempFolder.newFolder().getAbsolutePath(); String backups = tempFolder.newFolder().getAbsolutePath(); // we use the fs backend with small threshold here to test the behaviour with file // references, not self contained byte handles RocksDBStateBackend rdb = new RocksDBStateBackend( new FsStateBackend( new Path("file://" + backups).toUri(), fileSizeThreshold), incrementalCheckpoints); rdb.setDbStoragePath(rocksDb); this.stateBackend = rdb; }
env.setStateBackend(new FsStateBackend(statePath));
final String checkpointDir = params.get("checkpoint-dir"); boolean asyncCheckpoints = params.getBoolean("async-checkpoints", false); env.setStateBackend(new FsStateBackend(checkpointDir, asyncCheckpoints)); } else if ("rocks".equals(stateBackend)) { final String checkpointDir = params.get("checkpoint-dir");
case FILE: { String backups = tempFolder.newFolder().getAbsolutePath(); this.stateBackend = new FsStateBackend("file://" + backups, false); break; case FILE_ASYNC: { String backups = tempFolder.newFolder().getAbsolutePath(); this.stateBackend = new FsStateBackend("file://" + backups, true); break;
break; case "fs": stateBackend = new FsStateBackend(tmpPath); break; case "memory":
public static void main(String[] args) throws Exception { final ParameterTool pt = ParameterTool.fromArgs(args); final String checkpointDir = pt.getRequired("checkpoint.dir"); final StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment(); env.setStateBackend(new FsStateBackend(checkpointDir)); env.setRestartStrategy(RestartStrategies.noRestart()); env.enableCheckpointing(1000L); env.getConfig().disableGenericTypes(); env.addSource(new MySource()).uid("my-source") .keyBy(anInt -> 0) .map(new MyStatefulFunction()).uid("my-map") .addSink(new DiscardingSink<>()).uid("my-sink"); env.execute(); } }
STATE_BACKEND_FILE_ASYNC.defaultValue()); env.setStateBackend(new FsStateBackend(checkpointDir, asyncCheckpoints)); } else if ("rocks".equalsIgnoreCase(stateBackend)) { boolean incrementalCheckpoints = pt.getBoolean(
env.enableCheckpointing(200); env.setStateBackend(new FsStateBackend(tempCheckpointDir.getAbsoluteFile().toURI()));
env.setStateBackend(new FsStateBackend(checkpointDir, asyncCheckpoints)); } else if ("rocks".equals(stateBackend)) { boolean incrementalCheckpoints = pt.getBoolean("incrementalCheckpoints", false);
env.setStateBackend((StateBackend) new FsStateBackend(checkpointLocation.toURI()));