when(streamFactory.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE)).thenReturn(outputStream1, outputStream2); context.getRawOperatorStateOutput(); verify(streamFactory, times(2)).createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE);
.createCheckpointStateOutputStream(CheckpointedStateScope.SHARED); closeableRegistry.registerCloseable(outputStream);
private CheckpointStreamFactory.CheckpointStateOutputStream openAndRegisterNewStream() throws Exception { CheckpointStreamFactory.CheckpointStateOutputStream cout = streamFactory.createCheckpointStateOutputStream(checkpointId, checkpointTimestamp); closableRegistry.registerClosable(cout); return cout; }
private CheckpointStreamFactory.CheckpointStateOutputStream openAndRegisterNewStream() throws Exception { CheckpointStreamFactory.CheckpointStateOutputStream cout = streamFactory.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE); closableRegistry.registerCloseable(cout); return cout; }
@Nonnull static CheckpointStreamWithResultProvider createSimpleStream( @Nonnull long checkpointId, @Nonnull CheckpointedStateScope checkpointedStateScope, @Nonnull CheckpointStreamFactory primaryStreamFactory) throws IOException { CheckpointStreamFactory.CheckpointStateOutputStream primaryOut = primaryStreamFactory.createCheckpointStateOutputStream(checkpointId, checkpointedStateScope); return new CheckpointStreamWithResultProvider.PrimaryStreamOnly(primaryOut); }
private CheckpointStreamFactory.CheckpointStateOutputStream openAndRegisterNewStream() throws Exception { CheckpointStreamFactory.CheckpointStateOutputStream cout = streamFactory.createCheckpointStateOutputStream(checkpointId, CheckpointedStateScope.EXCLUSIVE); closableRegistry.registerCloseable(cout); return cout; }
@Nonnull static CheckpointStreamWithResultProvider createSimpleStream( @Nonnull CheckpointedStateScope checkpointedStateScope, @Nonnull CheckpointStreamFactory primaryStreamFactory) throws IOException { CheckpointStreamFactory.CheckpointStateOutputStream primaryOut = primaryStreamFactory.createCheckpointStateOutputStream(checkpointedStateScope); return new CheckpointStreamWithResultProvider.PrimaryStreamOnly(primaryOut); }
private CheckpointStreamFactory.CheckpointStateOutputStream openAndRegisterNewStream() throws Exception { CheckpointStreamFactory.CheckpointStateOutputStream cout = streamFactory.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE); closableRegistry.registerCloseable(cout); return cout; }
@Nonnull static CheckpointStreamWithResultProvider createSimpleStream( @Nonnull CheckpointedStateScope checkpointedStateScope, @Nonnull CheckpointStreamFactory primaryStreamFactory) throws IOException { CheckpointStreamFactory.CheckpointStateOutputStream primaryOut = primaryStreamFactory.createCheckpointStateOutputStream(checkpointedStateScope); return new CheckpointStreamWithResultProvider.PrimaryStreamOnly(primaryOut); }
private void openOutStream() throws Exception { out = streamFactory.createCheckpointStateOutputStream(checkpointId, CheckpointedStateScope.EXCLUSIVE); closeStreamOnCancelRegistry.registerCloseable(out); }
@Override public CheckpointStreamFactory.CheckpointStateOutputStream openIOHandle() throws Exception { if (checkStreamClosedAndDoTransitionToOpen()) { CheckpointStreamFactory.CheckpointStateOutputStream stream = streamFactory.createCheckpointStateOutputStream(checkpointId, timestamp); try { closeStreamOnCancelRegistry.registerClosable(stream); return stream; } catch (Exception ex) { open.set(false); throw ex; } } else { throw new IOException("Async snapshot: a checkpoint stream was already opened."); } }
@Nonnull static CheckpointStreamWithResultProvider createDuplicatingStream( @Nonnegative long checkpointId, @Nonnull CheckpointedStateScope checkpointedStateScope, @Nonnull CheckpointStreamFactory primaryStreamFactory, @Nonnull LocalRecoveryDirectoryProvider secondaryStreamDirProvider) throws IOException { CheckpointStreamFactory.CheckpointStateOutputStream primaryOut = primaryStreamFactory.createCheckpointStateOutputStream(checkpointedStateScope); try { File outFile = new File( secondaryStreamDirProvider.subtaskSpecificCheckpointDirectory(checkpointId), String.valueOf(UUID.randomUUID())); Path outPath = new Path(outFile.toURI()); CheckpointStreamFactory.CheckpointStateOutputStream secondaryOut = new FileBasedStateOutputStream(outPath.getFileSystem(), outPath); return new CheckpointStreamWithResultProvider.PrimaryAndSecondaryStream(primaryOut, secondaryOut); } catch (IOException secondaryEx) { LOG.warn("Exception when opening secondary/local checkpoint output stream. " + "Continue only with the primary stream.", secondaryEx); } return new CheckpointStreamWithResultProvider.PrimaryStreamOnly(primaryOut); }
@Nonnull static CheckpointStreamWithResultProvider createDuplicatingStream( @Nonnegative long checkpointId, @Nonnull CheckpointedStateScope checkpointedStateScope, @Nonnull CheckpointStreamFactory primaryStreamFactory, @Nonnull LocalRecoveryDirectoryProvider secondaryStreamDirProvider) throws IOException { CheckpointStreamFactory.CheckpointStateOutputStream primaryOut = primaryStreamFactory.createCheckpointStateOutputStream(checkpointedStateScope); try { File outFile = new File( secondaryStreamDirProvider.subtaskSpecificCheckpointDirectory(checkpointId), String.valueOf(UUID.randomUUID())); Path outPath = new Path(outFile.toURI()); CheckpointStreamFactory.CheckpointStateOutputStream secondaryOut = new FileBasedStateOutputStream(outPath.getFileSystem(), outPath); return new CheckpointStreamWithResultProvider.PrimaryAndSecondaryStream(primaryOut, secondaryOut); } catch (IOException secondaryEx) { LOG.warn("Exception when opening secondary/local checkpoint output stream. " + "Continue only with the primary stream.", secondaryEx); } return new CheckpointStreamWithResultProvider.PrimaryStreamOnly(primaryOut); }
@Nonnull static CheckpointStreamWithResultProvider createDuplicatingStream( @Nonnegative long checkpointId, @Nonnull CheckpointedStateScope checkpointedStateScope, @Nonnull CheckpointStreamFactory primaryStreamFactory, @Nonnull LocalRecoveryDirectoryProvider secondaryStreamDirProvider) throws IOException { CheckpointStreamFactory.CheckpointStateOutputStream primaryOut = primaryStreamFactory.createCheckpointStateOutputStream(checkpointId, checkpointedStateScope); try { File outFile = new File( secondaryStreamDirProvider.subtaskSpecificCheckpointDirectory(checkpointId), String.valueOf(UUID.randomUUID())); Path outPath = new Path(outFile.toURI()); CheckpointStreamFactory.CheckpointStateOutputStream secondaryOut = new FileBasedStateOutputStream(outPath.getFileSystem(), outPath); return new CheckpointStreamWithResultProvider.PrimaryAndSecondaryStream(primaryOut, secondaryOut); } catch (IOException secondaryEx) { LOG.warn("Exception when opening secondary/local checkpoint output stream. " + "Continue only with the primary stream.", secondaryEx); } return new CheckpointStreamWithResultProvider.PrimaryStreamOnly(primaryOut); }
@Override public void processElement(StreamRecord<IN> element) throws Exception { IN value = element.getValue(); // generate initial operator state if (out == null) { out = checkpointStreamFactory.createCheckpointStateOutputStream(0, 0); } serializer.serialize(value, new DataOutputViewStreamWrapper(out)); }
/** * 2) Open CheckpointStateOutputStream through the checkpointStreamFactory into which we will write. * * @throws Exception */ public void openCheckpointStream() throws Exception { Preconditions.checkArgument(outStream == null, "Output stream for snapshot is already set."); outStream = checkpointStreamFactory. createCheckpointStateOutputStream(checkpointId, checkpointTimeStamp); stateBackend.cancelStreamRegistry.registerClosable(outStream); outputView = new DataOutputViewStreamWrapper(outStream); }
streamFactory.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE); registerCloseableForCancellation(localOut);
streamFactory.createCheckpointStateOutputStream(CheckpointedStateScope.EXCLUSIVE); registerCloseableForCancellation(localOut);
/** * @deprecated Non-repartitionable operator state that has been deprecated. * Can be removed when we remove the APIs for non-repartitionable operator state. */ @SuppressWarnings("deprecation") @Deprecated @Override public StreamStateHandle snapshotLegacyOperatorState(long checkpointId, long timestamp, CheckpointOptions checkpointOptions) throws Exception { if (this instanceof StreamCheckpointedOperator) { CheckpointStreamFactory factory = getCheckpointStreamFactory(checkpointOptions); final CheckpointStreamFactory.CheckpointStateOutputStream outStream = factory.createCheckpointStateOutputStream(checkpointId, timestamp); getContainingTask().getCancelables().registerClosable(outStream); try { ((StreamCheckpointedOperator) this).snapshotState(outStream, checkpointId, timestamp); return outStream.closeAndGetHandle(); } finally { getContainingTask().getCancelables().unregisterClosable(outStream); outStream.close(); } } else { return null; } }
/** * This is public so that we can use it when restoring a legacy snapshot * in {@code AbstractStreamOperatorTestHarness}. */ public static KeyGroupsStateHandle convertKeyedBackendState( HashMap<String, KvStateSnapshot<?, ?, ?, ?>> oldKeyedState, int parallelInstanceIdx, long checkpointID) throws Exception { if (null != oldKeyedState) { CheckpointStreamFactory checkpointStreamFactory = new MemCheckpointStreamFactory(MAX_SIZE); CheckpointStreamFactory.CheckpointStateOutputStream keyedStateOut = checkpointStreamFactory.createCheckpointStateOutputStream(checkpointID, 0L); try { final long offset = keyedStateOut.getPos(); InstantiationUtil.serializeObject(keyedStateOut, oldKeyedState); StreamStateHandle streamStateHandle = keyedStateOut.closeAndGetHandle(); keyedStateOut = null; // makes IOUtils.closeQuietly(...) ignore this if (null != streamStateHandle) { KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(parallelInstanceIdx, parallelInstanceIdx, new long[]{offset}); return new MigrationKeyGroupStateHandle(keyGroupRangeOffsets, streamStateHandle); } } finally { IOUtils.closeQuietly(keyedStateOut); } } return null; }