operatorStateMetadata.put(DefaultOperatorStateBackend.DEFAULT_OPERATOR_STATE_NAME, metaInfo); KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(new KeyGroupRange(0, 0));
DataOutputView dov = new DataOutputViewStreamWrapper(out); KeyGroupRangeOffsets offsets = new KeyGroupRangeOffsets(i == 9 ? KeyGroupRange.EMPTY_KEY_GROUP_RANGE : new KeyGroupRange(prev, end)); prev = end + 1; for (int kg : offsets.getKeyGroupRange()) {
RocksDBFullSnapshotOperation( RocksDBKeyedStateBackend<K> stateBackend, CheckpointStreamFactory checkpointStreamFactory) { this.stateBackend = stateBackend; this.checkpointStreamFactory = checkpointStreamFactory; this.keyGroupRangeOffsets = new KeyGroupRangeOffsets(stateBackend.keyGroupRange); }
private static KeyedStateHandle deserializeRawKeyedStateHandle(DataInputStream dis) throws IOException { final int type = dis.readByte(); if (NULL_HANDLE == type) { return null; } else if (KEY_GROUPS_HANDLE == type) { int startKeyGroup = dis.readInt(); int numKeyGroups = dis.readInt(); KeyGroupRange keyGroupRange = KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1); long[] offsets = new long[numKeyGroups]; for (int i = 0; i < numKeyGroups; ++i) { offsets[i] = dis.readLong(); } KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets( keyGroupRange, offsets); StreamStateHandle stateHandle = deserializeStreamStateHandle(dis); return new KeyGroupsStateHandle(keyGroupRangeOffsets, stateHandle); } else { throw new IllegalStateException("Reading invalid RawKeyedStateHandle, type: " + type); } }
public KeyedStateCheckpointOutputStream( CheckpointStreamFactory.CheckpointStateOutputStream delegate, KeyGroupRange keyGroupRange) { super(delegate); Preconditions.checkNotNull(keyGroupRange); Preconditions.checkArgument(keyGroupRange != KeyGroupRange.EMPTY_KEY_GROUP_RANGE); this.currentKeyGroup = NO_CURRENT_KEY_GROUP; long[] emptyOffsets = new long[keyGroupRange.getNumberOfKeyGroups()]; // mark offsets as currently not set Arrays.fill(emptyOffsets, NO_OFFSET_SET); this.keyGroupRangeOffsets = new KeyGroupRangeOffsets(keyGroupRange, emptyOffsets); }
public KeyedStateCheckpointOutputStream( CheckpointStreamFactory.CheckpointStateOutputStream delegate, KeyGroupRange keyGroupRange) { super(delegate); Preconditions.checkNotNull(keyGroupRange); Preconditions.checkArgument(keyGroupRange != KeyGroupRange.EMPTY_KEY_GROUP_RANGE); this.currentKeyGroup = NO_CURRENT_KEY_GROUP; long[] emptyOffsets = new long[keyGroupRange.getNumberOfKeyGroups()]; // mark offsets as currently not set Arrays.fill(emptyOffsets, NO_OFFSET_SET); this.keyGroupRangeOffsets = new KeyGroupRangeOffsets(keyGroupRange, emptyOffsets); }
public KeyedStateCheckpointOutputStream( CheckpointStreamFactory.CheckpointStateOutputStream delegate, KeyGroupRange keyGroupRange) { super(delegate); Preconditions.checkNotNull(keyGroupRange); Preconditions.checkArgument(keyGroupRange != KeyGroupRange.EMPTY_KEY_GROUP_RANGE); this.currentKeyGroup = NO_CURRENT_KEY_GROUP; long[] emptyOffsets = new long[keyGroupRange.getNumberOfKeyGroups()]; // mark offsets as currently not set Arrays.fill(emptyOffsets, NO_OFFSET_SET); this.keyGroupRangeOffsets = new KeyGroupRangeOffsets(keyGroupRange, emptyOffsets); }
public KeyedStateCheckpointOutputStream( CheckpointStreamFactory.CheckpointStateOutputStream delegate, KeyGroupRange keyGroupRange) { super(delegate); Preconditions.checkNotNull(keyGroupRange); Preconditions.checkArgument(keyGroupRange != KeyGroupRange.EMPTY_KEY_GROUP_RANGE); this.currentKeyGroup = NO_CURRENT_KEY_GROUP; long[] emptyOffsets = new long[keyGroupRange.getNumberOfKeyGroups()]; // mark offsets as currently not set Arrays.fill(emptyOffsets, NO_OFFSET_SET); this.keyGroupRangeOffsets = new KeyGroupRangeOffsets(keyGroupRange, emptyOffsets); }
@VisibleForTesting public static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis) throws IOException { final int type = dis.readByte(); if (NULL_HANDLE == type) { return null; } else if (KEY_GROUPS_HANDLE == type) { int startKeyGroup = dis.readInt(); int numKeyGroups = dis.readInt(); KeyGroupRange keyGroupRange = KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1); long[] offsets = new long[numKeyGroups]; for (int i = 0; i < numKeyGroups; ++i) { offsets[i] = dis.readLong(); } KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets( keyGroupRange, offsets); StreamStateHandle stateHandle = deserializeStreamStateHandle(dis); return new KeyGroupsStateHandle(keyGroupRangeOffsets, stateHandle); } else { throw new IllegalStateException("Reading invalid KeyedStateHandle, type: " + type); } }
@VisibleForTesting public static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis) throws IOException { final int type = dis.readByte(); if (NULL_HANDLE == type) { return null; } else if (KEY_GROUPS_HANDLE == type) { int startKeyGroup = dis.readInt(); int numKeyGroups = dis.readInt(); KeyGroupRange keyGroupRange = KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1); long[] offsets = new long[numKeyGroups]; for (int i = 0; i < numKeyGroups; ++i) { offsets[i] = dis.readLong(); } KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets( keyGroupRange, offsets); StreamStateHandle stateHandle = deserializeStreamStateHandle(dis); return new KeyGroupsStateHandle(keyGroupRangeOffsets, stateHandle); } else { throw new IllegalStateException("Reading invalid KeyedStateHandle, type: " + type); } }
@VisibleForTesting public static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis) throws IOException { final int type = dis.readByte(); if (NULL_HANDLE == type) { return null; } else if (KEY_GROUPS_HANDLE == type) { int startKeyGroup = dis.readInt(); int numKeyGroups = dis.readInt(); KeyGroupRange keyGroupRange = KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1); long[] offsets = new long[numKeyGroups]; for (int i = 0; i < numKeyGroups; ++i) { offsets[i] = dis.readLong(); } KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets( keyGroupRange, offsets); StreamStateHandle stateHandle = deserializeStreamStateHandle(dis); return new KeyGroupsStateHandle(keyGroupRangeOffsets, stateHandle); } else { throw new IllegalStateException("Reading invalid KeyedStateHandle, type: " + type); } }
@VisibleForTesting public static KeyedStateHandle deserializeKeyedStateHandle(DataInputStream dis) throws IOException { final int type = dis.readByte(); if (NULL_HANDLE == type) { return null; } else if (KEY_GROUPS_HANDLE == type) { int startKeyGroup = dis.readInt(); int numKeyGroups = dis.readInt(); KeyGroupRange keyGroupRange = KeyGroupRange.of(startKeyGroup, startKeyGroup + numKeyGroups - 1); long[] offsets = new long[numKeyGroups]; for (int i = 0; i < numKeyGroups; ++i) { offsets[i] = dis.readLong(); } KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets( keyGroupRange, offsets); StreamStateHandle stateHandle = deserializeStreamStateHandle(dis); return new KeyGroupsStateHandle(keyGroupRangeOffsets, stateHandle); } else { throw new IllegalStateException("Reading invalid KeyedStateHandle, type: " + type); } }
offsets[i] = dis.readLong(); KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets( keyGroupRange, offsets); StreamStateHandle stateHandle = deserializeStreamStateHandle(dis);
offsets[i] = dis.readLong(); KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets( keyGroupRange, offsets); StreamStateHandle stateHandle = deserializeStreamStateHandle(dis);
@Override public KeyGroupsStateHandle performOperation() throws Exception { long asyncStartTime = System.currentTimeMillis(); CheckpointStreamFactory.CheckpointStateOutputStream stream = getIoHandle(); DataOutputViewStreamWrapper outView = new DataOutputViewStreamWrapper(stream); serializationProxy.write(outView); long[] keyGroupRangeOffsets = new long[keyGroupRange.getNumberOfKeyGroups()]; for (int keyGroupPos = 0; keyGroupPos < keyGroupRange.getNumberOfKeyGroups(); ++keyGroupPos) { int keyGroupId = keyGroupRange.getKeyGroupId(keyGroupPos); keyGroupRangeOffsets[keyGroupPos] = stream.getPos(); outView.writeInt(keyGroupId); for (Map.Entry<String, StateTable<K, ?, ?>> kvState : stateTables.entrySet()) { outView.writeShort(kVStateToId.get(kvState.getKey())); cowStateStableSnapshots.get(kvState.getValue()).writeMappingsInKeyGroup(outView, keyGroupId); } } final StreamStateHandle streamStateHandle = closeStreamAndGetStateHandle(); if (asynchronousSnapshots) { LOG.info("Heap backend snapshot ({}, asynchronous part) in thread {} took {} ms.", streamFactory, Thread.currentThread(), (System.currentTimeMillis() - asyncStartTime)); } if (streamStateHandle == null) { return null; } KeyGroupRangeOffsets offsets = new KeyGroupRangeOffsets(keyGroupRange, keyGroupRangeOffsets); final KeyGroupsStateHandle keyGroupsStateHandle = new KeyGroupsStateHandle(offsets, streamStateHandle); return keyGroupsStateHandle; } };
/** * Returns a key-group range with offsets which is the intersection of the internal key-group range with the given * key-group range. * * @param keyGroupRange Key-group range to intersect with the internal key-group range. * @return The key-group range with offsets for the intersection of the internal key-group range with the given * key-group range. */ public KeyGroupRangeOffsets getIntersection(KeyGroupRange keyGroupRange) { Preconditions.checkNotNull(keyGroupRange); KeyGroupRange intersection = this.keyGroupRange.getIntersection(keyGroupRange); long[] subOffsets = new long[intersection.getNumberOfKeyGroups()]; if(subOffsets.length > 0) { System.arraycopy( offsets, computeKeyGroupIndex(intersection.getStartKeyGroup()), subOffsets, 0, subOffsets.length); } return new KeyGroupRangeOffsets(intersection, subOffsets); }
/** * Returns a key-group range with offsets which is the intersection of the internal key-group range with the given * key-group range. * * @param keyGroupRange Key-group range to intersect with the internal key-group range. * @return The key-group range with offsets for the intersection of the internal key-group range with the given * key-group range. */ public KeyGroupRangeOffsets getIntersection(KeyGroupRange keyGroupRange) { Preconditions.checkNotNull(keyGroupRange); KeyGroupRange intersection = this.keyGroupRange.getIntersection(keyGroupRange); long[] subOffsets = new long[intersection.getNumberOfKeyGroups()]; if(subOffsets.length > 0) { System.arraycopy( offsets, computeKeyGroupIndex(intersection.getStartKeyGroup()), subOffsets, 0, subOffsets.length); } return new KeyGroupRangeOffsets(intersection, subOffsets); }
/** * Returns a key-group range with offsets which is the intersection of the internal key-group range with the given * key-group range. * * @param keyGroupRange Key-group range to intersect with the internal key-group range. * @return The key-group range with offsets for the intersection of the internal key-group range with the given * key-group range. */ public KeyGroupRangeOffsets getIntersection(KeyGroupRange keyGroupRange) { Preconditions.checkNotNull(keyGroupRange); KeyGroupRange intersection = this.keyGroupRange.getIntersection(keyGroupRange); long[] subOffsets = new long[intersection.getNumberOfKeyGroups()]; if(subOffsets.length > 0) { System.arraycopy( offsets, computeKeyGroupIndex(intersection.getStartKeyGroup()), subOffsets, 0, subOffsets.length); } return new KeyGroupRangeOffsets(intersection, subOffsets); }
/** * Returns a key-group range with offsets which is the intersection of the internal key-group range with the given * key-group range. * * @param keyGroupRange Key-group range to intersect with the internal key-group range. * @return The key-group range with offsets for the intersection of the internal key-group range with the given * key-group range. */ public KeyGroupRangeOffsets getIntersection(KeyGroupRange keyGroupRange) { Preconditions.checkNotNull(keyGroupRange); KeyGroupRange intersection = this.keyGroupRange.getIntersection(keyGroupRange); long[] subOffsets = new long[intersection.getNumberOfKeyGroups()]; if(subOffsets.length > 0) { System.arraycopy( offsets, computeKeyGroupIndex(intersection.getStartKeyGroup()), subOffsets, 0, subOffsets.length); } return new KeyGroupRangeOffsets(intersection, subOffsets); }
/** * This is public so that we can use it when restoring a legacy snapshot * in {@code AbstractStreamOperatorTestHarness}. */ public static KeyGroupsStateHandle convertKeyedBackendState( HashMap<String, KvStateSnapshot<?, ?, ?, ?>> oldKeyedState, int parallelInstanceIdx, long checkpointID) throws Exception { if (null != oldKeyedState) { CheckpointStreamFactory checkpointStreamFactory = new MemCheckpointStreamFactory(MAX_SIZE); CheckpointStreamFactory.CheckpointStateOutputStream keyedStateOut = checkpointStreamFactory.createCheckpointStateOutputStream(checkpointID, 0L); try { final long offset = keyedStateOut.getPos(); InstantiationUtil.serializeObject(keyedStateOut, oldKeyedState); StreamStateHandle streamStateHandle = keyedStateOut.closeAndGetHandle(); keyedStateOut = null; // makes IOUtils.closeQuietly(...) ignore this if (null != streamStateHandle) { KeyGroupRangeOffsets keyGroupRangeOffsets = new KeyGroupRangeOffsets(parallelInstanceIdx, parallelInstanceIdx, new long[]{offset}); return new MigrationKeyGroupStateHandle(keyGroupRangeOffsets, streamStateHandle); } } finally { IOUtils.closeQuietly(keyedStateOut); } } return null; }