public void restoreStateForKeyGroup(DataInputViewStreamWrapper stream, int keyGroupIdx, ClassLoader userCodeClassLoader) throws IOException, ClassNotFoundException { int noOfTimerServices = stream.readInt(); for (int i = 0; i < noOfTimerServices; i++) { String serviceName = stream.readUTF(); HeapInternalTimerService<K, N> timerService = timerServices.get(serviceName); if (timerService == null) { timerService = new HeapInternalTimerService<>( totalKeyGroups, localKeyGroupRange, keyContext, processingTimeService); timerServices.put(serviceName, timerService); } timerService.restoreTimersForKeyGroup(stream, keyGroupIdx, userCodeClassLoader); } }
private void restoreFromLegacyWindowOperator(DataInputViewStreamWrapper in) throws IOException { Preconditions.checkArgument(legacyWindowOperatorType == LegacyWindowOperatorType.NONE); int numWatermarkTimers = in.readInt(); this.restoredFromLegacyEventTimeTimers = new PriorityQueue<>(Math.max(numWatermarkTimers, 1)); int numProcessingTimeTimers = in.readInt(); this.restoredFromLegacyProcessingTimeTimers = new PriorityQueue<>(Math.max(numProcessingTimeTimers, 1)); int numProcessingTimeTimerTimestamp = in.readInt(); for (int i = 0; i < numProcessingTimeTimerTimestamp; i++) { in.readLong(); in.readInt();
private void seekNextStateId(boolean metaFollows) throws Exception { int stateId = compressedInputView.readShort(); String stateName = stateIdMapping.get(stateId); if (metaFollows) { stateId = END_OF_KEY_GROUP_MARK & stateId; } while (!stateFilter.filter(stateName) && stateId != END_OF_KEY_GROUP_MARK) { final int keySize = compressedInputView.readInt(); final byte keyByte0 = compressedInputView.readByte(); compressedInputView.skip(keySize - 1); final int valueSize = compressedInputView.readInt(); compressedInputView.skip(valueSize); if (hasMetaDataFollowsFlag(keyByte0)) { stateId = END_OF_KEY_GROUP_MARK & compressedInputView.readShort(); stateName = stateIdMapping.get(stateId); } } this.stateId = stateId; this.stateName = stateName; }
int noOfSplits = div.readInt(); for (int i = 0; i < noOfSplits; i++) { FileInputSplit split = (FileInputSplit) ois.readObject();
private void restoreFromLegacyAlignedWindowOperator(DataInputViewStreamWrapper in) throws IOException { Preconditions.checkArgument(legacyWindowOperatorType != LegacyWindowOperatorType.NONE); final long nextEvaluationTime = in.readLong(); final long nextSlideTime = in.readLong(); validateMagicNumber(BEGIN_OF_STATE_MAGIC_NUMBER, in.readInt()); restoredFromLegacyAlignedOpRecords = new PriorityQueue<>(42, new Comparator<StreamRecord<IN>>() { @Override public int compare(StreamRecord<IN> o1, StreamRecord<IN> o2) { return Long.compare(o1.getTimestamp(), o2.getTimestamp()); } } ); switch (legacyWindowOperatorType) { case FAST_ACCUMULATING: restoreElementsFromLegacyAccumulatingAlignedWindowOperator(in, nextSlideTime); break; case FAST_AGGREGATING: restoreElementsFromLegacyAggregatingAlignedWindowOperator(in, nextSlideTime); break; } if (LOG.isDebugEnabled()) { LOG.debug("{} (taskIdx={}) restored {} events from legacy {}.", getClass().getSimpleName(), getRuntimeContext().getIndexOfThisSubtask(), restoredFromLegacyAlignedOpRecords.size(), legacyWindowOperatorType); } }
int sizeOfEventTimeTimers = stream.readInt(); if (sizeOfEventTimeTimers > 0) { Set<InternalTimer<K, N>> eventTimers = getEventTimeTimerSetForKeyGroup(keyGroupIdx); int sizeOfProcessingTimeTimers = stream.readInt(); if (sizeOfProcessingTimeTimers > 0) { Set<InternalTimer<K, N>> processingTimers = getProcessingTimeTimerSetForKeyGroup(keyGroupIdx);
int writtenKeyGroupIndex = inputView.readInt(); Preconditions.checkState(writtenKeyGroupIndex == group, "Unexpected key-group in restore."); int stateId = kgCompressionInView.readInt(); KeyedStateDescriptor descriptor = keyedStatesById.get(stateId); HeapStateStorage stateStorage = (HeapStateStorage) stateStorages.get(descriptor.getName()); int stateId = kgCompressionInView.readInt(); SubKeyedStateDescriptor descriptor = subKeyedStatesById.get(stateId); HeapStateStorage stateStorage = (HeapStateStorage) stateStorages.get(descriptor.getName());
private void readStateHandleStateData( FSDataInputStream fsDataInputStream, DataInputViewStreamWrapper inView, KeyGroupRangeOffsets keyGroupOffsets, Map<Integer, StateMetaInfoSnapshot> kvStatesById, int numStates, int readVersion, boolean isCompressed) throws IOException { final StreamCompressionDecorator streamCompressionDecorator = isCompressed ? SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE; for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) { int keyGroupIndex = groupOffset.f0; long offset = groupOffset.f1; // Check that restored key groups all belong to the backend. Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend."); fsDataInputStream.seek(offset); int writtenKeyGroupIndex = inView.readInt(); Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex, "Unexpected key-group in restore."); try (InputStream kgCompressionInStream = streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) { readKeyGroupStateData( kgCompressionInStream, kvStatesById, keyGroupIndex, numStates, readVersion); } } }
private void readStateHandleStateData( FSDataInputStream fsDataInputStream, DataInputViewStreamWrapper inView, KeyGroupRangeOffsets keyGroupOffsets, Map<Integer, StateMetaInfoSnapshot> kvStatesById, int numStates, int readVersion, boolean isCompressed) throws IOException { final StreamCompressionDecorator streamCompressionDecorator = isCompressed ? SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE; for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) { int keyGroupIndex = groupOffset.f0; long offset = groupOffset.f1; // Check that restored key groups all belong to the backend. Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend."); fsDataInputStream.seek(offset); int writtenKeyGroupIndex = inView.readInt(); Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex, "Unexpected key-group in restore."); try (InputStream kgCompressionInStream = streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) { readKeyGroupStateData( kgCompressionInStream, kvStatesById, keyGroupIndex, numStates, readVersion); } } }
int numberEntries = inputView.readInt(); for (int i = 0; i < numberEntries; i++) { KEY key = keySerializer.deserialize(inputView);
int writtenKeyGroupIndex = inView.readInt();
int writtenKeyGroupIndex = inView.readInt();