/** * Removes the entropy marker string from the path, if the given file system is an * entropy-injecting file system (implements {@link EntropyInjectingFileSystem}) and * the entropy marker key is present. Otherwise, this returns the path as is. * * @param path The path to filter. * @return The path without the marker string. */ public static Path removeEntropyMarkerIfPresent(FileSystem fs, Path path) { final EntropyInjectingFileSystem efs = getEntropyFs(fs); if (efs == null) { return path; } else { try { return resolveEntropy(path, efs, false); } catch (IOException e) { // this should never happen, because the path was valid before and we only remove characters. // rethrow to silence the compiler throw new FlinkRuntimeException(e.getMessage(), e); } } }
@Test public void testCreateEntropyAwarePlainFs() throws Exception { File folder = TMP_FOLDER.newFolder(); Path path = new Path(Path.fromLocalFile(folder), "_entropy_/file"); OutputStreamAndPath out = EntropyInjector.createEntropyAware( LocalFileSystem.getSharedInstance(), path, WriteMode.NO_OVERWRITE); out.stream().close(); assertEquals(path, out.path()); assertTrue(new File (new File(folder, "_entropy_"), "file").exists()); }
@Test public void testEmptyPath() throws Exception { EntropyInjectingFileSystem efs = new TestEntropyInjectingFs("test", "ignored"); Path path = new Path("hdfs://localhost:12345"); assertEquals(path, EntropyInjector.resolveEntropy(path, efs, true)); assertEquals(path, EntropyInjector.resolveEntropy(path, efs, false)); }
public FsCheckpointStorageLocation( FileSystem fileSystem, Path checkpointDir, Path sharedStateDir, Path taskOwnedStateDir, CheckpointStorageLocationReference reference, int fileStateSizeThreshold) { super(fileSystem, checkpointDir, sharedStateDir, fileStateSizeThreshold); checkArgument(fileStateSizeThreshold >= 0); this.fileSystem = checkNotNull(fileSystem); this.checkpointDirectory = checkNotNull(checkpointDir); this.sharedStateDirectory = checkNotNull(sharedStateDir); this.taskOwnedStateDirectory = checkNotNull(taskOwnedStateDir); this.reference = checkNotNull(reference); // the metadata file should not have entropy in its path Path metadataDir = EntropyInjector.removeEntropyMarkerIfPresent(fileSystem, checkpointDir); this.metadataFilePath = new Path(metadataDir, AbstractFsCheckpointStorage.METADATA_FILE_NAME); this.fileStateSizeThreshold = fileStateSizeThreshold; }
@Test public void testCreateEntropyAwareEntropyFs() throws Exception { File folder = TMP_FOLDER.newFolder(); Path path = new Path(Path.fromLocalFile(folder), "_entropy_/file"); Path pathWithEntropy = new Path(Path.fromLocalFile(folder), "test-entropy/file"); FileSystem fs = new TestEntropyInjectingFs("_entropy_", "test-entropy"); OutputStreamAndPath out = EntropyInjector.createEntropyAware(fs, path, WriteMode.NO_OVERWRITE); out.stream().close(); assertEquals(new Path(Path.fromLocalFile(folder), "test-entropy/file"), out.path()); assertTrue(new File (new File(folder, "test-entropy"), "file").exists()); }
@Test public void testFullUriNonMatching() throws Exception { EntropyInjectingFileSystem efs = new TestEntropyInjectingFs("_entropy_key_", "ignored"); Path path = new Path("s3://hugo@myawesomehost:55522/path/to/the/file"); assertEquals(path, EntropyInjector.resolveEntropy(path, efs, true)); assertEquals(path, EntropyInjector.resolveEntropy(path, efs, false)); }
public FsCheckpointStorageLocation( FileSystem fileSystem, Path checkpointDir, Path sharedStateDir, Path taskOwnedStateDir, CheckpointStorageLocationReference reference, int fileStateSizeThreshold) { super(fileSystem, checkpointDir, sharedStateDir, fileStateSizeThreshold); checkArgument(fileStateSizeThreshold >= 0); this.fileSystem = checkNotNull(fileSystem); this.checkpointDirectory = checkNotNull(checkpointDir); this.sharedStateDirectory = checkNotNull(sharedStateDir); this.taskOwnedStateDirectory = checkNotNull(taskOwnedStateDir); this.reference = checkNotNull(reference); // the metadata file should not have entropy in its path Path metadataDir = EntropyInjector.removeEntropyMarkerIfPresent(fileSystem, checkpointDir); this.metadataFilePath = new Path(metadataDir, AbstractFsCheckpointStorage.METADATA_FILE_NAME); this.fileStateSizeThreshold = fileStateSizeThreshold; }
@Test public void testWithSafetyNet() throws Exception { final String entropyKey = "__ekey__"; final String entropyValue = "abc"; final File folder = TMP_FOLDER.newFolder(); final Path path = new Path(Path.fromLocalFile(folder), entropyKey + "/path/"); final Path pathWithEntropy = new Path(Path.fromLocalFile(folder), entropyValue + "/path/"); TestEntropyInjectingFs efs = new TestEntropyInjectingFs(entropyKey, entropyValue); FSDataOutputStream out; FileSystemSafetyNet.initializeSafetyNetForThread(); FileSystem fs = FileSystemSafetyNet.wrapWithSafetyNetWhenActivated(efs); try { OutputStreamAndPath streamAndPath = EntropyInjector.createEntropyAware( fs, path, WriteMode.NO_OVERWRITE); out = streamAndPath.stream(); assertEquals(pathWithEntropy, streamAndPath.path()); } finally { FileSystemSafetyNet.closeSafetyNetAndGuardedResourcesForThread(); } // check that the safety net closed the stream try { out.write(42); out.flush(); fail("stream should be already close and hence fail with an exception"); } catch (IOException ignored) {} }
/** * Handles entropy injection across regular and entropy-aware file systems. * * <p>If the given file system is entropy-aware (a implements {@link EntropyInjectingFileSystem}), * then this method replaces the entropy marker in the path with random characters. * The entropy marker is defined by {@link EntropyInjectingFileSystem#getEntropyInjectionKey()}. * * <p>If the given file system does not implement {@code EntropyInjectingFileSystem}, * then this method delegates to {@link FileSystem#create(Path, WriteMode)} and * returns the same path in the resulting {@code OutputStreamAndPath}. */ public static OutputStreamAndPath createEntropyAware( FileSystem fs, Path path, WriteMode writeMode) throws IOException { // check and possibly inject entropy into the path final EntropyInjectingFileSystem efs = getEntropyFs(fs); final Path processedPath = efs == null ? path : resolveEntropy(path, efs, true); // create the stream on the original file system to let the safety net // take its effect final FSDataOutputStream out = fs.create(processedPath, writeMode); return new OutputStreamAndPath(out, processedPath); }
@Test public void testPathOnlyNonMatching() throws Exception { EntropyInjectingFileSystem efs = new TestEntropyInjectingFs("_entropy_key_", "ignored"); Path path = new Path("/path/file"); assertEquals(path, EntropyInjector.resolveEntropy(path, efs, true)); assertEquals(path, EntropyInjector.resolveEntropy(path, efs, false)); }
private void createStream() throws IOException { Exception latestException = null; for (int attempt = 0; attempt < 10; attempt++) { try { OutputStreamAndPath streamAndPath = EntropyInjector.createEntropyAware( fs, createStatePath(), WriteMode.NO_OVERWRITE); this.outStream = streamAndPath.stream(); this.statePath = streamAndPath.path(); return; } catch (Exception e) { latestException = e; } } throw new IOException("Could not open output stream for state backend", latestException); } }
/** * Removes the entropy marker string from the path, if the given file system is an * entropy-injecting file system (implements {@link EntropyInjectingFileSystem}) and * the entropy marker key is present. Otherwise, this returns the path as is. * * @param path The path to filter. * @return The path without the marker string. */ public static Path removeEntropyMarkerIfPresent(FileSystem fs, Path path) { final EntropyInjectingFileSystem efs = getEntropyFs(fs); if (efs == null) { return path; } else { try { return resolveEntropy(path, efs, false); } catch (IOException e) { // this should never happen, because the path was valid before and we only remove characters. // rethrow to silence the compiler throw new FlinkRuntimeException(e.getMessage(), e); } } }
@Test public void testFullUriMatching() throws Exception { EntropyInjectingFileSystem efs = new TestEntropyInjectingFs("s0mek3y", "12345678"); Path path = new Path("s3://hugo@myawesomehost:55522/path/s0mek3y/the/file"); assertEquals(new Path("s3://hugo@myawesomehost:55522/path/12345678/the/file"), EntropyInjector.resolveEntropy(path, efs, true)); assertEquals(new Path("s3://hugo@myawesomehost:55522/path/the/file"), EntropyInjector.resolveEntropy(path, efs, false)); }
private void createStream() throws IOException { Exception latestException = null; for (int attempt = 0; attempt < 10; attempt++) { try { OutputStreamAndPath streamAndPath = EntropyInjector.createEntropyAware( fs, createStatePath(), WriteMode.NO_OVERWRITE); this.outStream = streamAndPath.stream(); this.statePath = streamAndPath.path(); return; } catch (Exception e) { latestException = e; } } throw new IOException("Could not open output stream for state backend", latestException); } }
/** * Handles entropy injection across regular and entropy-aware file systems. * * <p>If the given file system is entropy-aware (a implements {@link EntropyInjectingFileSystem}), * then this method replaces the entropy marker in the path with random characters. * The entropy marker is defined by {@link EntropyInjectingFileSystem#getEntropyInjectionKey()}. * * <p>If the given file system does not implement {@code EntropyInjectingFileSystem}, * then this method delegates to {@link FileSystem#create(Path, WriteMode)} and * returns the same path in the resulting {@code OutputStreamAndPath}. */ public static OutputStreamAndPath createEntropyAware( FileSystem fs, Path path, WriteMode writeMode) throws IOException { // check and possibly inject entropy into the path final EntropyInjectingFileSystem efs = getEntropyFs(fs); final Path processedPath = efs == null ? path : resolveEntropy(path, efs, true); // create the stream on the original file system to let the safety net // take its effect final FSDataOutputStream out = fs.create(processedPath, writeMode); return new OutputStreamAndPath(out, processedPath); }
@Test public void testEntropyNotFullSegment() throws Exception { EntropyInjectingFileSystem efs = new TestEntropyInjectingFs("_entropy_key_", "pqr"); Path path = new Path("s3://myhost:122/entropy-_entropy_key_-suffix/file"); assertEquals(new Path("s3://myhost:122/entropy-pqr-suffix/file"), EntropyInjector.resolveEntropy(path, efs, true)); assertEquals(new Path("s3://myhost:122/entropy--suffix/file"), EntropyInjector.resolveEntropy(path, efs, false)); }
@Test public void testPathOnlyMatching() throws Exception { EntropyInjectingFileSystem efs = new TestEntropyInjectingFs("_entropy_key_", "xyzz"); Path path = new Path("/path/_entropy_key_/file"); assertEquals(new Path("/path/xyzz/file"), EntropyInjector.resolveEntropy(path, efs, true)); assertEquals(new Path("/path/file"), EntropyInjector.resolveEntropy(path, efs, false)); }