@Override public CheckpointStreamFactory resolveCheckpointStorageLocation( long checkpointId, CheckpointStorageLocationReference reference) throws IOException { if (reference.isDefaultReference()) { // default reference, construct the default location for that particular checkpoint final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId); return new FsCheckpointStorageLocation( fileSystem, checkpointDir, sharedStateDirectory, taskOwnedStateDirectory, reference, fileSizeThreshold); } else { // location encoded in the reference final Path path = decodePathFromReference(reference); return new FsCheckpointStorageLocation( path.getFileSystem(), path, path, path, reference, fileSizeThreshold); } }
@Override protected CheckpointStorageLocation createSavepointLocation(FileSystem fs, Path location) throws IOException { final CheckpointStorageLocationReference reference = encodePathAsReference(location); return new FsCheckpointStorageLocation(fs, location, location, location, reference, fileSizeThreshold); } }
@Override public CheckpointStorage createCheckpointStorage(JobID jobId) throws IOException { checkNotNull(jobId, "jobId"); return new FsCheckpointStorage(getCheckpointPath(), getSavepointPath(), jobId, getMinFileSizeThreshold()); }
@Override public CheckpointStorageLocation initializeLocationForCheckpoint(long checkpointId) throws IOException { checkArgument(checkpointId >= 0); // prepare all the paths needed for the checkpoints final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId); // create the checkpoint exclusive directory fileSystem.mkdirs(checkpointDir); return new FsCheckpointStorageLocation( fileSystem, checkpointDir, sharedStateDirectory, taskOwnedStateDirectory, CheckpointStorageLocationReference.getDefault(), fileSizeThreshold); }
public FsCheckpointStorage( FileSystem fs, Path checkpointBaseDirectory, @Nullable Path defaultSavepointDirectory, JobID jobId, int fileSizeThreshold) throws IOException { super(jobId, defaultSavepointDirectory); checkArgument(fileSizeThreshold >= 0); this.fileSystem = checkNotNull(fs); this.checkpointsDirectory = getCheckpointDirectoryForJob(checkpointBaseDirectory, jobId); this.sharedStateDirectory = new Path(checkpointsDirectory, CHECKPOINT_SHARED_STATE_DIR); this.taskOwnedStateDirectory = new Path(checkpointsDirectory, CHECKPOINT_TASK_OWNED_STATE_DIR); this.fileSizeThreshold = fileSizeThreshold; // initialize the dedicated directories fileSystem.mkdirs(checkpointsDirectory); fileSystem.mkdirs(sharedStateDirectory); fileSystem.mkdirs(taskOwnedStateDirectory); }
@Override public CheckpointStorage createCheckpointStorage(JobID jobId) throws IOException { checkNotNull(jobId, "jobId"); return new FsCheckpointStorage(getCheckpointPath(), getSavepointPath(), jobId, getMinFileSizeThreshold()); }
@Override public CheckpointStorageLocation initializeLocationForCheckpoint(long checkpointId) throws IOException { checkArgument(checkpointId >= 0); // prepare all the paths needed for the checkpoints final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId); // create the checkpoint exclusive directory fileSystem.mkdirs(checkpointDir); return new FsCheckpointStorageLocation( fileSystem, checkpointDir, sharedStateDirectory, taskOwnedStateDirectory, CheckpointStorageLocationReference.getDefault(), fileSizeThreshold); }
public FsCheckpointStorage( FileSystem fs, Path checkpointBaseDirectory, @Nullable Path defaultSavepointDirectory, JobID jobId, int fileSizeThreshold) throws IOException { super(jobId, defaultSavepointDirectory); checkArgument(fileSizeThreshold >= 0); this.fileSystem = checkNotNull(fs); this.checkpointsDirectory = getCheckpointDirectoryForJob(checkpointBaseDirectory, jobId); this.sharedStateDirectory = new Path(checkpointsDirectory, CHECKPOINT_SHARED_STATE_DIR); this.taskOwnedStateDirectory = new Path(checkpointsDirectory, CHECKPOINT_TASK_OWNED_STATE_DIR); this.fileSizeThreshold = fileSizeThreshold; // initialize the dedicated directories fileSystem.mkdirs(checkpointsDirectory); fileSystem.mkdirs(sharedStateDirectory); fileSystem.mkdirs(taskOwnedStateDirectory); }
@Override public CheckpointStreamFactory resolveCheckpointStorageLocation( long checkpointId, CheckpointStorageLocationReference reference) throws IOException { if (reference.isDefaultReference()) { // default reference, construct the default location for that particular checkpoint final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId); return new FsCheckpointStorageLocation( fileSystem, checkpointDir, sharedStateDirectory, taskOwnedStateDirectory, reference, fileSizeThreshold); } else { // location encoded in the reference final Path path = decodePathFromReference(reference); return new FsCheckpointStorageLocation( path.getFileSystem(), path, path, path, reference, fileSizeThreshold); } }
@Override protected CheckpointStorageLocation createSavepointLocation(FileSystem fs, Path location) throws IOException { final CheckpointStorageLocationReference reference = encodePathAsReference(location); return new FsCheckpointStorageLocation(fs, location, location, location, reference, fileSizeThreshold); } }
@Override public CheckpointStorage createCheckpointStorage(JobID jobId) throws IOException { checkNotNull(jobId, "jobId"); return new FsCheckpointStorage( getCheckpointPath(), getSavepointPath(), jobId, createCheckpointSubDirs.getOrDefault(CheckpointingOptions.CHCKPOINTS_CREATE_SUBDIRS.defaultValue()), getMinFileSizeThreshold()); }
@Override public CheckpointStorageLocation initializeLocationForCheckpoint(long checkpointId) throws IOException { checkArgument(checkpointId >= 0); // prepare all the paths needed for the checkpoints final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId); // create the checkpoint exclusive directory fileSystem.mkdirs(checkpointDir); return new FsCheckpointStorageLocation( fileSystem, checkpointDir, exclusiveStateDirectory, sharedStateDirectory, taskOwnedStateDirectory, CheckpointStorageLocationReference.getDefault(), fileSizeThreshold); }
public FsCheckpointStorage( Path checkpointBaseDirectory, @Nullable Path defaultSavepointDirectory, JobID jobId, boolean createCheckpointSubDir, int fileSizeThreshold) throws IOException { super(jobId, defaultSavepointDirectory); checkArgument(fileSizeThreshold >= 0); this.fileSystem = checkpointBaseDirectory.getFileSystem(); this.checkpointsDirectory = createCheckpointSubDir ? getCheckpointDirectoryForJob(checkpointBaseDirectory, jobId) : checkpointBaseDirectory; this.exclusiveStateDirectory = new Path(checkpointsDirectory, CHECKPOINT_EXCLUSICE_STATE_DIR); this.sharedStateDirectory = new Path(checkpointsDirectory, CHECKPOINT_SHARED_STATE_DIR); this.taskOwnedStateDirectory = new Path(checkpointsDirectory, CHECKPOINT_TASK_OWNED_STATE_DIR); this.fileSizeThreshold = fileSizeThreshold; this.stateSubDirs = 0; // initialize the dedicated directories fileSystem.mkdirs(checkpointsDirectory); fileSystem.mkdirs(exclusiveStateDirectory); fileSystem.mkdirs(sharedStateDirectory); fileSystem.mkdirs(taskOwnedStateDirectory); }
final Path checkpointDir = createCheckpointDirectory(checkpointsDirectory, checkpointId); final Path path = decodePathFromReference(reference);
@Override protected CheckpointStorageLocation createSavepointLocation(FileSystem fs, Path location) throws IOException { final CheckpointStorageLocationReference reference = encodePathAsReference(location); return new FsCheckpointStorageLocation(fs, location, location, location, location, reference, fileSizeThreshold); } }