@Test public void testWithSafetyNet() throws Exception { final String entropyKey = "__ekey__"; final String entropyValue = "abc"; final File folder = TMP_FOLDER.newFolder(); final Path path = new Path(Path.fromLocalFile(folder), entropyKey + "/path/"); final Path pathWithEntropy = new Path(Path.fromLocalFile(folder), entropyValue + "/path/"); TestEntropyInjectingFs efs = new TestEntropyInjectingFs(entropyKey, entropyValue); FSDataOutputStream out; FileSystemSafetyNet.initializeSafetyNetForThread(); FileSystem fs = FileSystemSafetyNet.wrapWithSafetyNetWhenActivated(efs); try { OutputStreamAndPath streamAndPath = EntropyInjector.createEntropyAware( fs, path, WriteMode.NO_OVERWRITE); out = streamAndPath.stream(); assertEquals(pathWithEntropy, streamAndPath.path()); } finally { FileSystemSafetyNet.closeSafetyNetAndGuardedResourcesForThread(); } // check that the safety net closed the stream try { out.write(42); out.flush(); fail("stream should be already close and hence fail with an exception"); } catch (IOException ignored) {} }
FileSystemSafetyNet.initializeSafetyNetForThread(); FileSystemSafetyNet.closeSafetyNetAndGuardedResourcesForThread();
@Override public void run() { // set safety net from the task's context for checkpointing thread LOG.debug("Creating FileSystem stream leak safety net for {}", Thread.currentThread().getName()); FileSystemSafetyNet.setSafetyNetCloseableRegistryForThread(safetyNetCloseableRegistry); try { boolean success = invokable.triggerCheckpoint(checkpointMetaData, checkpointOptions); if (!success) { checkpointResponder.declineCheckpoint( getJobID(), getExecutionId(), checkpointID, new CheckpointDeclineTaskNotReadyException(taskName)); } } catch (Throwable t) { if (getExecutionState() == ExecutionState.RUNNING) { failExternally(new Exception( "Error while triggering checkpoint " + checkpointID + " for " + taskNameWithSubtask, t)); } else { LOG.debug("Encountered error while triggering checkpoint {} for " + "{} ({}) while being not in state running.", checkpointID, taskNameWithSubtask, executionId, t); } } finally { FileSystemSafetyNet.setSafetyNetCloseableRegistryForThread(null); } } };
FileSystemSafetyNet.getSafetyNetCloseableRegistryForThread();
/** * Returns a reference to the {@link FileSystem} instance for accessing the local file system. * * @return a reference to the {@link FileSystem} instance for accessing the local file system. */ public static FileSystem getLocalFileSystem() { return FileSystemSafetyNet.wrapWithSafetyNetWhenActivated(LocalFileSystem.getSharedInstance()); }
FileSystemSafetyNet.initializeSafetyNetForThread(); FileSystemSafetyNet.closeSafetyNetAndGuardedResourcesForThread();
@Override public void run() { // set safety net from the task's context for checkpointing thread LOG.debug("Creating FileSystem stream leak safety net for {}", Thread.currentThread().getName()); FileSystemSafetyNet.setSafetyNetCloseableRegistryForThread(safetyNetCloseableRegistry); try { boolean success = invokable.triggerCheckpoint(checkpointMetaData, checkpointOptions); if (!success) { checkpointResponder.declineCheckpoint( getJobID(), getExecutionId(), checkpointID, new CheckpointDeclineTaskNotReadyException(taskName)); } } catch (Throwable t) { if (getExecutionState() == ExecutionState.RUNNING) { failExternally(new Exception( "Error while triggering checkpoint " + checkpointID + " for " + taskNameWithSubtask, t)); } else { LOG.debug("Encountered error while triggering checkpoint {} for " + "{} ({}) while being not in state running.", checkpointID, taskNameWithSubtask, executionId, t); } } finally { FileSystemSafetyNet.setSafetyNetCloseableRegistryForThread(null); } } };
FileSystemSafetyNet.getSafetyNetCloseableRegistryForThread();
/** * Returns a reference to the {@link FileSystem} instance for accessing the * file system identified by the given {@link URI}. * * @param uri * the {@link URI} identifying the file system * @return a reference to the {@link FileSystem} instance for accessing the file system identified by the given * {@link URI}. * @throws IOException * thrown if a reference to the file system instance could not be obtained */ public static FileSystem get(URI uri) throws IOException { return FileSystemSafetyNet.wrapWithSafetyNetWhenActivated(getUnguardedFileSystem(uri)); }
FileSystemSafetyNet.initializeSafetyNetForThread(); FileSystemSafetyNet.closeSafetyNetAndGuardedResourcesForThread();
@Override public void run() { // set safety net from the task's context for checkpointing thread LOG.debug("Creating FileSystem stream leak safety net for {}", Thread.currentThread().getName()); FileSystemSafetyNet.setSafetyNetCloseableRegistryForThread(safetyNetCloseableRegistry); try { boolean success = invokable.triggerCheckpoint(checkpointMetaData, checkpointOptions); if (!success) { checkpointResponder.declineCheckpoint( getJobID(), getExecutionId(), checkpointID, new CheckpointDeclineTaskNotReadyException(taskName)); } } catch (Throwable t) { if (getExecutionState() == ExecutionState.RUNNING) { failExternally(new Exception( "Error while triggering checkpoint " + checkpointID + " for " + taskNameWithSubtask, t)); } else { LOG.debug("Encountered error while triggering checkpoint {} for " + "{} ({}) while being not in state running.", checkpointID, taskNameWithSubtask, executionId, t); } } finally { FileSystemSafetyNet.setSafetyNetCloseableRegistryForThread(null); } } };
FileSystemSafetyNet.getSafetyNetCloseableRegistryForThread();
/** * Returns a reference to the {@link FileSystem} instance for accessing the local file system. * * @return a reference to the {@link FileSystem} instance for accessing the local file system. */ public static FileSystem getLocalFileSystem() { return FileSystemSafetyNet.wrapWithSafetyNetWhenActivated(LocalFileSystem.getSharedInstance()); }
FileSystemSafetyNet.initializeSafetyNetForThread(); FileSystemSafetyNet.closeSafetyNetAndGuardedResourcesForThread();
@Override public void run() { // set safety net from the task's context for checkpointing thread LOG.debug("Creating FileSystem stream leak safety net for {}", Thread.currentThread().getName()); FileSystemSafetyNet.setSafetyNetCloseableRegistryForThread(safetyNetCloseableRegistry); try { boolean success = statefulTask.triggerCheckpoint(checkpointMetaData, checkpointOptions); if (!success) { checkpointResponder.declineCheckpoint( getJobID(), getExecutionId(), checkpointID, new CheckpointDeclineTaskNotReadyException(taskName)); } } catch (Throwable t) { if (getExecutionState() == ExecutionState.RUNNING) { failExternally(new Exception( "Error while triggering checkpoint " + checkpointID + " for " + taskNameWithSubtask, t)); } else { LOG.debug("Encountered error while triggering checkpoint {} for " + "{} ({}) while being not in state running.", checkpointID, taskNameWithSubtask, executionId, t); } } finally { FileSystemSafetyNet.setSafetyNetCloseableRegistryForThread(null); } } };
final String taskName = taskNameWithSubtask; final SafetyNetCloseableRegistry safetyNetCloseableRegistry = FileSystemSafetyNet.getSafetyNetCloseableRegistryForThread(); Runnable runnable = new Runnable() { @Override
/** * Returns a reference to the {@link FileSystem} instance for accessing the * file system identified by the given {@link URI}. * * @param uri * the {@link URI} identifying the file system * @return a reference to the {@link FileSystem} instance for accessing the file system identified by the given * {@link URI}. * @throws IOException * thrown if a reference to the file system instance could not be obtained */ public static FileSystem get(URI uri) throws IOException { return FileSystemSafetyNet.wrapWithSafetyNetWhenActivated(getUnguardedFileSystem(uri)); }
/** * Returns a reference to the {@link FileSystem} instance for accessing the local file system. * * @return a reference to the {@link FileSystem} instance for accessing the local file system. */ public static FileSystem getLocalFileSystem() { return FileSystemSafetyNet.wrapWithSafetyNetWhenActivated(LocalFileSystem.getSharedInstance()); }
/** * Returns a reference to the {@link FileSystem} instance for accessing the * file system identified by the given {@link URI}. * * @param uri * the {@link URI} identifying the file system * @return a reference to the {@link FileSystem} instance for accessing the file system identified by the given * {@link URI}. * @throws IOException * thrown if a reference to the file system instance could not be obtained */ public static FileSystem get(URI uri) throws IOException { return FileSystemSafetyNet.wrapWithSafetyNetWhenActivated(getUnguardedFileSystem(uri)); }