/** * Removes the entropy marker string from the path, if the given file system is an * entropy-injecting file system (implements {@link EntropyInjectingFileSystem}) and * the entropy marker key is present. Otherwise, this returns the path as is. * * @param path The path to filter. * @return The path without the marker string. */ public static Path removeEntropyMarkerIfPresent(FileSystem fs, Path path) { final EntropyInjectingFileSystem efs = getEntropyFs(fs); if (efs == null) { return path; } else { try { return resolveEntropy(path, efs, false); } catch (IOException e) { // this should never happen, because the path was valid before and we only remove characters. // rethrow to silence the compiler throw new FlinkRuntimeException(e.getMessage(), e); } } }
/** * Handles entropy injection across regular and entropy-aware file systems. * * <p>If the given file system is entropy-aware (a implements {@link EntropyInjectingFileSystem}), * then this method replaces the entropy marker in the path with random characters. * The entropy marker is defined by {@link EntropyInjectingFileSystem#getEntropyInjectionKey()}. * * <p>If the given file system does not implement {@code EntropyInjectingFileSystem}, * then this method delegates to {@link FileSystem#create(Path, WriteMode)} and * returns the same path in the resulting {@code OutputStreamAndPath}. */ public static OutputStreamAndPath createEntropyAware( FileSystem fs, Path path, WriteMode writeMode) throws IOException { // check and possibly inject entropy into the path final EntropyInjectingFileSystem efs = getEntropyFs(fs); final Path processedPath = efs == null ? path : resolveEntropy(path, efs, true); // create the stream on the original file system to let the safety net // take its effect final FSDataOutputStream out = fs.create(processedPath, writeMode); return new OutputStreamAndPath(out, processedPath); }
@Test public void testFullUriNonMatching() throws Exception { EntropyInjectingFileSystem efs = new TestEntropyInjectingFs("_entropy_key_", "ignored"); Path path = new Path("s3://hugo@myawesomehost:55522/path/to/the/file"); assertEquals(path, EntropyInjector.resolveEntropy(path, efs, true)); assertEquals(path, EntropyInjector.resolveEntropy(path, efs, false)); }
@Test public void testPathOnlyNonMatching() throws Exception { EntropyInjectingFileSystem efs = new TestEntropyInjectingFs("_entropy_key_", "ignored"); Path path = new Path("/path/file"); assertEquals(path, EntropyInjector.resolveEntropy(path, efs, true)); assertEquals(path, EntropyInjector.resolveEntropy(path, efs, false)); }
@Test public void testEmptyPath() throws Exception { EntropyInjectingFileSystem efs = new TestEntropyInjectingFs("test", "ignored"); Path path = new Path("hdfs://localhost:12345"); assertEquals(path, EntropyInjector.resolveEntropy(path, efs, true)); assertEquals(path, EntropyInjector.resolveEntropy(path, efs, false)); }
@Test public void testFullUriMatching() throws Exception { EntropyInjectingFileSystem efs = new TestEntropyInjectingFs("s0mek3y", "12345678"); Path path = new Path("s3://hugo@myawesomehost:55522/path/s0mek3y/the/file"); assertEquals(new Path("s3://hugo@myawesomehost:55522/path/12345678/the/file"), EntropyInjector.resolveEntropy(path, efs, true)); assertEquals(new Path("s3://hugo@myawesomehost:55522/path/the/file"), EntropyInjector.resolveEntropy(path, efs, false)); }
@Test public void testPathOnlyMatching() throws Exception { EntropyInjectingFileSystem efs = new TestEntropyInjectingFs("_entropy_key_", "xyzz"); Path path = new Path("/path/_entropy_key_/file"); assertEquals(new Path("/path/xyzz/file"), EntropyInjector.resolveEntropy(path, efs, true)); assertEquals(new Path("/path/file"), EntropyInjector.resolveEntropy(path, efs, false)); }
@Test public void testEntropyNotFullSegment() throws Exception { EntropyInjectingFileSystem efs = new TestEntropyInjectingFs("_entropy_key_", "pqr"); Path path = new Path("s3://myhost:122/entropy-_entropy_key_-suffix/file"); assertEquals(new Path("s3://myhost:122/entropy-pqr-suffix/file"), EntropyInjector.resolveEntropy(path, efs, true)); assertEquals(new Path("s3://myhost:122/entropy--suffix/file"), EntropyInjector.resolveEntropy(path, efs, false)); }
/** * Removes the entropy marker string from the path, if the given file system is an * entropy-injecting file system (implements {@link EntropyInjectingFileSystem}) and * the entropy marker key is present. Otherwise, this returns the path as is. * * @param path The path to filter. * @return The path without the marker string. */ public static Path removeEntropyMarkerIfPresent(FileSystem fs, Path path) { final EntropyInjectingFileSystem efs = getEntropyFs(fs); if (efs == null) { return path; } else { try { return resolveEntropy(path, efs, false); } catch (IOException e) { // this should never happen, because the path was valid before and we only remove characters. // rethrow to silence the compiler throw new FlinkRuntimeException(e.getMessage(), e); } } }
/** * Handles entropy injection across regular and entropy-aware file systems. * * <p>If the given file system is entropy-aware (a implements {@link EntropyInjectingFileSystem}), * then this method replaces the entropy marker in the path with random characters. * The entropy marker is defined by {@link EntropyInjectingFileSystem#getEntropyInjectionKey()}. * * <p>If the given file system does not implement {@code EntropyInjectingFileSystem}, * then this method delegates to {@link FileSystem#create(Path, WriteMode)} and * returns the same path in the resulting {@code OutputStreamAndPath}. */ public static OutputStreamAndPath createEntropyAware( FileSystem fs, Path path, WriteMode writeMode) throws IOException { // check and possibly inject entropy into the path final EntropyInjectingFileSystem efs = getEntropyFs(fs); final Path processedPath = efs == null ? path : resolveEntropy(path, efs, true); // create the stream on the original file system to let the safety net // take its effect final FSDataOutputStream out = fs.create(processedPath, writeMode); return new OutputStreamAndPath(out, processedPath); }