@Override public boolean hasNext() { if (null != currentStateHandle && currentOffsetsIterator.hasNext()) { return true; } closeCurrentStream(); while (stateHandleIterator.hasNext()) { currentStateHandle = stateHandleIterator.next(); if (currentStateHandle.getKeyGroupRange().getNumberOfKeyGroups() > 0) { currentOffsetsIterator = currentStateHandle.getGroupRangeOffsets().iterator(); return true; } } return false; }
/** * Restore one key groups state handle. */ private void restoreKeyGroupsInStateHandle() throws IOException, StateMigrationException, RocksDBException { try { currentStateHandleInStream = currentKeyGroupsStateHandle.openInputStream(); rocksDBKeyedStateBackend.cancelStreamRegistry.registerCloseable(currentStateHandleInStream); currentStateHandleInView = new DataInputViewStreamWrapper(currentStateHandleInStream); restoreKVStateMetaData(); restoreKVStateData(); } finally { if (rocksDBKeyedStateBackend.cancelStreamRegistry.unregisterCloseable(currentStateHandleInStream)) { IOUtils.closeQuietly(currentStateHandleInStream); } } }
Collections.singletonList(new KeyGroupsStateHandle(keyGroupRangeOffsets, state));
private static void serializeRawKeyedStateHandle( KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException { if (stateHandle == null) { dos.writeByte(NULL_HANDLE); } else if (stateHandle instanceof KeyGroupsStateHandle) { KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle; dos.writeByte(KEY_GROUPS_HANDLE); dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup()); dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups()); for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) { dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup)); } serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos); } else { throw new IllegalStateException("Unknown RawKeyedStateHandle type: " + stateHandle.getClass()); } }
FSDataInputStream fsDataInputStream = keyGroupsStateHandle.openInputStream(); cancelStreamRegistry.registerCloseable(fsDataInputStream); fsDataInputStream, inView, keyGroupsStateHandle.getGroupRangeOffsets(), kvStatesById, restoredMetaInfos.size(), serializationProxy.getReadVersion(),
private boolean openIfNeeded() throws Exception { if (stateHandleInStream == null) { LOGGER.debug("Opening {}", keyGroupsStateHandle.getDelegateStateHandle()); stateHandleInStream = keyGroupsStateHandle.openInputStream(); final KeyedBackendSerializationProxy<?> serializationProxy = StateMetadataUtils .getKeyedBackendSerializationProxy(keyGroupsStateHandle); this.stateIdMapping = StateMetadataUtils.getStateIdMapping(serializationProxy); final StreamCompressionDecorator streamCompressionDecorator = StateMetadataUtils .getCompressionDecorator(serializationProxy); final KeyGroupRangeOffsets rangeOffsets = keyGroupsStateHandle.getGroupRangeOffsets(); LOGGER.debug("{}", rangeOffsets); offsetsIt = new ValidOffsetsIterator(rangeOffsets); hasNext = seekNextOffset(); if (hasNext) { final InputStream compressedInputStream = streamCompressionDecorator .decorateWithCompression(stateHandleInStream); compressedInputView = new DataInputViewStreamWrapper(compressedInputStream); seekNextStateId(false); } } return hasNext; }
for (Tuple2<Integer, Long> keyGroupOffset : currentKeyGroupsStateHandle.getGroupRangeOffsets()) { int keyGroup = keyGroupOffset.f0;
@Override public void close() throws IOException { if (stateHandleInStream != null) { LOGGER.debug("Closing {}", this.keyGroupsStateHandle.getDelegateStateHandle()); stateHandleInStream.close(); } }
@VisibleForTesting public static void serializeKeyedStateHandle( KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException { if (stateHandle == null) { dos.writeByte(NULL_HANDLE); } else if (stateHandle instanceof KeyGroupsStateHandle) { KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle; dos.writeByte(KEY_GROUPS_HANDLE); dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup()); dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups()); for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) { dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup)); } serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos); } else { throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass()); } }
FSDataInputStream fsDataInputStream = keyGroupsStateHandle.openInputStream(); cancelStreamRegistry.registerCloseable(fsDataInputStream); fsDataInputStream, inView, keyGroupsStateHandle.getGroupRangeOffsets(), kvStatesById, restoredMetaInfos.size(), serializationProxy.getReadVersion(),
for (Tuple2<Integer, Long> keyGroupOffset : currentKeyGroupsStateHandle.getGroupRangeOffsets()) { int keyGroup = keyGroupOffset.f0;
@VisibleForTesting public static void serializeKeyedStateHandle( KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException { if (stateHandle == null) { dos.writeByte(NULL_HANDLE); } else if (stateHandle instanceof KeyGroupsStateHandle) { KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle; dos.writeByte(KEY_GROUPS_HANDLE); dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup()); dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups()); for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) { dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup)); } serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos); } else { throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass()); } }
FSDataInputStream fsDataInputStream = keyGroupsStateHandle.openInputStream(); cancelStreamRegistry.registerClosable(fsDataInputStream); for (Tuple2<Integer, Long> groupOffset : keyGroupsStateHandle.getGroupRangeOffsets()) { int keyGroupIndex = groupOffset.f0; long offset = groupOffset.f1;
new KeyGroupsStateHandle(offsets, new ByteStateHandleCloseChecking("kg-" + i, out.toByteArray()));
@Override public boolean hasNext() { if (null != currentStateHandle && currentOffsetsIterator.hasNext()) { return true; } closeCurrentStream(); while (stateHandleIterator.hasNext()) { currentStateHandle = stateHandleIterator.next(); if (currentStateHandle.getKeyGroupRange().getNumberOfKeyGroups() > 0) { currentOffsetsIterator = currentStateHandle.getGroupRangeOffsets().iterator(); return true; } } return false; }
for (Tuple2<Integer, Long> keyGroupOffset : currentKeyGroupsStateHandle.getGroupRangeOffsets()) { int keyGroup = keyGroupOffset.f0;
/** * Restore one key groups state handle. */ private void restoreKeyGroupsInStateHandle() throws IOException, StateMigrationException, RocksDBException { try { currentStateHandleInStream = currentKeyGroupsStateHandle.openInputStream(); rocksDBKeyedStateBackend.cancelStreamRegistry.registerCloseable(currentStateHandleInStream); currentStateHandleInView = new DataInputViewStreamWrapper(currentStateHandleInStream); restoreKVStateMetaData(); restoreKVStateData(); } finally { if (rocksDBKeyedStateBackend.cancelStreamRegistry.unregisterCloseable(currentStateHandleInStream)) { IOUtils.closeQuietly(currentStateHandleInStream); } } }
@VisibleForTesting public static void serializeKeyedStateHandle( KeyedStateHandle stateHandle, DataOutputStream dos) throws IOException { if (stateHandle == null) { dos.writeByte(NULL_HANDLE); } else if (stateHandle instanceof KeyGroupsStateHandle) { KeyGroupsStateHandle keyGroupsStateHandle = (KeyGroupsStateHandle) stateHandle; dos.writeByte(KEY_GROUPS_HANDLE); dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getStartKeyGroup()); dos.writeInt(keyGroupsStateHandle.getKeyGroupRange().getNumberOfKeyGroups()); for (int keyGroup : keyGroupsStateHandle.getKeyGroupRange()) { dos.writeLong(keyGroupsStateHandle.getOffsetForKeyGroup(keyGroup)); } serializeStreamStateHandle(keyGroupsStateHandle.getDelegateStateHandle(), dos); } else { throw new IllegalStateException("Unknown KeyedStateHandle type: " + stateHandle.getClass()); } }
FSDataInputStream fsDataInputStream = keyGroupsStateHandle.openInputStream(); cancelStreamRegistry.registerCloseable(fsDataInputStream); SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE; for (Tuple2<Integer, Long> groupOffset : keyGroupsStateHandle.getGroupRangeOffsets()) { int keyGroupIndex = groupOffset.f0; long offset = groupOffset.f1;
/** * * @param keyGroupRange a key group range to intersect. * @return key-group state over a range that is the intersection between this handle's key-group range and the * provided key-group range. */ public KeyGroupsStateHandle getIntersection(KeyGroupRange keyGroupRange) { return new KeyGroupsStateHandle(groupRangeOffsets.getIntersection(keyGroupRange), stateHandle); }