@Override protected AbstractCloseableRegistry<Closeable, Object> createRegistry() { return new CloseableRegistry(); }
protected CloseableIterable<StatePartitionStreamProvider> rawOperatorStateInputs( Iterator<StateObjectCollection<OperatorStateHandle>> restoreStateAlternatives) { if (restoreStateAlternatives.hasNext()) { Collection<OperatorStateHandle> rawOperatorState = restoreStateAlternatives.next(); // TODO currently this does not support local state recovery, so we expect there is only one handle. Preconditions.checkState( !restoreStateAlternatives.hasNext(), "Local recovery is currently not implemented for raw operator state, but found state alternative."); if (rawOperatorState != null) { return new CloseableIterable<StatePartitionStreamProvider>() { final CloseableRegistry closeableRegistry = new CloseableRegistry(); @Override public void close() throws IOException { closeableRegistry.close(); } @Nonnull @Override public Iterator<StatePartitionStreamProvider> iterator() { return new OperatorStateStreamIterator( DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME, rawOperatorState.iterator(), closeableRegistry); } }; } } return CloseableIterable.empty(); }
protected CloseableIterable<KeyGroupStatePartitionStreamProvider> rawKeyedStateInputs( Iterator<StateObjectCollection<KeyedStateHandle>> restoreStateAlternatives) { if (restoreStateAlternatives.hasNext()) { Collection<KeyedStateHandle> rawKeyedState = restoreStateAlternatives.next(); // TODO currently this does not support local state recovery, so we expect there is only one handle. Preconditions.checkState( !restoreStateAlternatives.hasNext(), "Local recovery is currently not implemented for raw keyed state, but found state alternative."); if (rawKeyedState != null) { Collection<KeyGroupsStateHandle> keyGroupsStateHandles = transform(rawKeyedState); final CloseableRegistry closeableRegistry = new CloseableRegistry(); return new CloseableIterable<KeyGroupStatePartitionStreamProvider>() { @Override public void close() throws IOException { closeableRegistry.close(); } @Override public Iterator<KeyGroupStatePartitionStreamProvider> iterator() { return new KeyGroupStreamIterator(keyGroupsStateHandles.iterator(), closeableRegistry); } }; } } return CloseableIterable.empty(); }
@Before public void setUp() throws Exception { CloseableRegistry closableRegistry = new CloseableRegistry(); CheckpointStreamFactory streamFactory = new MemCheckpointStreamFactory(1024); KeyGroupRange keyGroupRange = new KeyGroupRange(0, 2); this.snapshotContext = new StateSnapshotContextSynchronousImpl(42, 4711, streamFactory, keyGroupRange, closableRegistry); }
this.config.setOperatorID(operatorID); this.executionConfig = env.getExecutionConfig(); this.closableRegistry = new CloseableRegistry(); this.checkpointLock = new Object();
/** * Tests if there is an exception if all restore attempts are exhausted and failed. */ @Test public void testExceptionThrownIfAllRestoresFailed() throws Exception { CloseableRegistry closeableRegistry = new CloseableRegistry(); OperatorStateHandle firstFailHandle = mock(OperatorStateHandle.class); OperatorStateHandle secondFailHandle = mock(OperatorStateHandle.class); OperatorStateHandle thirdFailHandle = mock(OperatorStateHandle.class); List<StateObjectCollection<OperatorStateHandle>> sortedRestoreOptions = Arrays.asList( new StateObjectCollection<>(Collections.singletonList(firstFailHandle)), new StateObjectCollection<>(Collections.singletonList(secondFailHandle)), new StateObjectCollection<>(Collections.singletonList(thirdFailHandle))); BackendRestorerProcedure<OperatorStateBackend, OperatorStateHandle> restorerProcedure = new BackendRestorerProcedure<>(backendSupplier, closeableRegistry, "test op state backend"); try { restorerProcedure.createAndRestore(sortedRestoreOptions); Assert.fail(); } catch (Exception ignore) { } verify(firstFailHandle).openInputStream(); verify(secondFailHandle).openInputStream(); verify(thirdFailHandle).openInputStream(); }
CloseableRegistry closeableRegistry = new CloseableRegistry(); OneShotLatch waitForBlock = new OneShotLatch(); OneShotLatch unblock = new OneShotLatch();
Whitebox.setInternalState(streamTask, "lock", new Object()); Whitebox.setInternalState(streamTask, "operatorChain", operatorChain); Whitebox.setInternalState(streamTask, "cancelables", new CloseableRegistry()); Whitebox.setInternalState(streamTask, "asyncOperationsThreadPool", newDirectExecutorService()); Whitebox.setInternalState(streamTask, "configuration", new StreamConfig(new Configuration()));
Whitebox.setInternalState(streamTask, "lock", new Object()); Whitebox.setInternalState(streamTask, "operatorChain", operatorChain); Whitebox.setInternalState(streamTask, "cancelables", new CloseableRegistry()); Whitebox.setInternalState(streamTask, "configuration", new StreamConfig(new Configuration())); Whitebox.setInternalState(streamTask, "checkpointStorage", new MemoryBackendCheckpointStorage(new JobID(), null, null, Integer.MAX_VALUE));
Whitebox.setInternalState(streamTask, "lock", new Object()); Whitebox.setInternalState(streamTask, "operatorChain", operatorChain); Whitebox.setInternalState(streamTask, "cancelables", new CloseableRegistry()); Whitebox.setInternalState(streamTask, "configuration", new StreamConfig(new Configuration())); Whitebox.setInternalState(streamTask, "asyncOperationsThreadPool", Executors.newCachedThreadPool());
final CloseableRegistry closeableRegistry = new CloseableRegistry();
Whitebox.setInternalState(streamTask, "lock", new Object()); Whitebox.setInternalState(streamTask, "operatorChain", operatorChain); Whitebox.setInternalState(streamTask, "cancelables", new CloseableRegistry()); Whitebox.setInternalState(streamTask, "asyncOperationsThreadPool", Executors.newFixedThreadPool(1)); Whitebox.setInternalState(streamTask, "configuration", new StreamConfig(new Configuration()));
Whitebox.setInternalState(streamTask, "lock", new Object()); Whitebox.setInternalState(streamTask, "operatorChain", operatorChain); Whitebox.setInternalState(streamTask, "cancelables", new CloseableRegistry()); Whitebox.setInternalState(streamTask, "asyncOperationsThreadPool", executor); Whitebox.setInternalState(streamTask, "configuration", new StreamConfig(new Configuration()));
/** * Checks that the state snapshot context is closed after a successful snapshot operation. */ @Test public void testSnapshotMethod() throws Exception { final long checkpointId = 42L; final long timestamp = 1L; final CloseableRegistry closeableRegistry = new CloseableRegistry(); StateSnapshotContextSynchronousImpl context = spy(new StateSnapshotContextSynchronousImpl(0L, 0L)); whenNew(StateSnapshotContextSynchronousImpl.class).withAnyArguments().thenReturn(context); StreamTask<Void, AbstractStreamOperator<Void>> containingTask = mock(StreamTask.class); when(containingTask.getCancelables()).thenReturn(closeableRegistry); AbstractStreamOperator<Void> operator = mock(AbstractStreamOperator.class); when(operator.snapshotState(anyLong(), anyLong(), any(CheckpointOptions.class), any(CheckpointStreamFactory.class))).thenCallRealMethod(); doReturn(containingTask).when(operator).getContainingTask(); operator.snapshotState( checkpointId, timestamp, CheckpointOptions.forCheckpointWithDefaultLocation(), new MemCheckpointStreamFactory(Integer.MAX_VALUE)); verify(context).close(); }
final CloseableRegistry closeableRegistry = new CloseableRegistry();
CloseableRegistry closeableRegistry = new CloseableRegistry();
public void testRestoreProcedureOrderAndFailure() throws Exception { CloseableRegistry closeableRegistry = new CloseableRegistry(); CheckpointStreamFactory checkpointStreamFactory = new MemCheckpointStreamFactory(1024);
CloseableRegistry closeableRegistry = new CloseableRegistry();
this.writtenOperatorStates = new HashSet<>(); this.closableRegistry = new CloseableRegistry();
public DefaultOperatorStateBackend( ClassLoader userClassLoader, ExecutionConfig executionConfig, boolean asynchronousSnapshots) throws IOException { this.closeStreamOnCancelRegistry = new CloseableRegistry(); this.userClassloader = Preconditions.checkNotNull(userClassLoader); this.executionConfig = executionConfig; this.javaSerializer = new JavaSerializer<>(); this.registeredStates = new HashMap<>(); this.asynchronousSnapshots = asynchronousSnapshots; this.accessedStatesByName = new HashMap<>(); this.restoredStateMetaInfos = new HashMap<>(); }