public static <T> T deserializeFromByteArray(TypeSerializer<T> serializer, T reuse, byte[] buf) throws IOException { if (buf == null) { throw new NullPointerException("Byte array to deserialize from must not be null."); } DataInputViewStreamWrapper inputViewWrapper = new DataInputViewStreamWrapper(new ByteArrayInputStream(buf)); return serializer.deserialize(reuse, inputViewWrapper); }
currentStateHandleInStream.seek(offset); try (InputStream compressedKgIn = keygroupStreamCompressionDecorator.decorateWithCompression(currentStateHandleInStream)) { DataInputViewStreamWrapper compressedKgInputView = new DataInputViewStreamWrapper(compressedKgIn); int kvStateId = compressedKgInputView.readShort(); ColumnFamilyHandle handle = currentStateHandleKVStateColumnFamilies.get(kvStateId); & compressedKgInputView.readShort(); if (END_OF_KEY_GROUP_MARK == kvStateId) { keyGroupHasMoreKeys = false;
private void restoreState() throws Exception { LOGGER.info("Restore siddhi state"); final Iterator<byte[]> siddhiState = siddhiRuntimeState.get().iterator(); if (siddhiState.hasNext()) { // TODO this.siddhiRuntime.restore(siddhiState.next()); } LOGGER.info("Restore queued records state"); final Iterator<byte[]> queueState = queuedRecordsState.get().iterator(); if (queueState.hasNext()) { final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(queueState.next()); final DataInputViewStreamWrapper dataInputView = new DataInputViewStreamWrapper(byteArrayInputStream); try { this.priorityQueue = restoreQueueState(dataInputView); } finally { dataInputView.close(); byteArrayInputStream.close(); } } }
DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(fsDataInputStream); int writtenKeyGroupIndex = inView.readInt(); new DataInputViewStreamWrapper(kgCompressionInStream); int kvStateId = kgCompressionInView.readShort(); StateTable<K, ?, ?> stateTable = stateTables.get(kvStatesById.get(kvStateId));
new DataInputViewStreamWrapper(inputStream); int writtenKeyGroupIndex = inputView.readInt(); Preconditions.checkState(writtenKeyGroupIndex == group, "Unexpected key-group in restore."); streamCompressionDecorator.decorateWithCompression(inputStream)) { DataInputViewStreamWrapper kgCompressionInView = new DataInputViewStreamWrapper(kgCompressionInStream); int stateId = kgCompressionInView.readInt(); KeyedStateDescriptor descriptor = keyedStatesById.get(stateId); HeapStateStorage stateStorage = (HeapStateStorage) stateStorages.get(descriptor.getName()); int stateId = kgCompressionInView.readInt(); SubKeyedStateDescriptor descriptor = subKeyedStatesById.get(stateId); HeapStateStorage stateStorage = (HeapStateStorage) stateStorages.get(descriptor.getName());
DataInputViewStreamWrapper inViewWrapper = new DataInputViewStreamWrapper(inWithPos)) { TypeSerializerSnapshot fieldSerializerConfigSnapshot; for (int i = 0; i < numFields; i++) { fieldName = inViewWrapper.readUTF(); TypeSerializerSnapshot registeredSubclassSerializerConfigSnapshot; for (int i = 0; i < numRegisteredSubclasses; i++) { registeredSubclassname = inViewWrapper.readUTF(); try { registeredSubclass = Class.forName(registeredSubclassname, true, getUserCodeClassLoader()); TypeSerializerSnapshot cachedSubclassSerializerConfigSnapshot; for (int i = 0; i < numCachedSubclassSerializers; i++) { cachedSubclassname = inViewWrapper.readUTF(); try { cachedSubclass = Class.forName(cachedSubclassname, true, getUserCodeClassLoader());
final DataInputViewStreamWrapper div = new DataInputViewStreamWrapper(in); int noOfSplits = div.readInt(); for (int i = 0; i < noOfSplits; i++) { FileInputSplit split = (FileInputSplit) ois.readObject(); div.close();
private UV deserializeUserValue(byte[] rawValueBytes) throws IOException { ByteArrayInputStreamWithPos bais = new ByteArrayInputStreamWithPos(rawValueBytes); DataInputViewStreamWrapper in = new DataInputViewStreamWrapper(bais); boolean isNull = in.readBoolean(); return isNull ? null : userValueSerializer.deserialize(in); }
public static <K, N> Pair<K, N> getDeserializedKeyAndNamespace( byte[] serializedBytes, TypeSerializer<K> keySerializer, TypeSerializer<N> namespaceSerializer, int serializedStateNameLenght) throws IOException { ByteArrayInputStreamWithPos inputStream = new ByteArrayInputStreamWithPos(serializedBytes); DataInputViewStreamWrapper inputView = new DataInputViewStreamWrapper(inputStream); K key = getDeserializedSingleKey(inputView, keySerializer, serializedStateNameLenght); inputView.skipBytesToRead(KEY_PREFIX_BYTE_LENGTH); N namespace = namespaceSerializer.deserialize(inputView); return new DefaultPair<>(key, namespace); }
@Override public void restoreState(FSDataInputStream in) throws Exception { super.restoreState(in); DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(in); final long nextEvaluationTime = inView.readLong(); final long nextSlideTime = inView.readLong(); AbstractKeyedTimePanes<IN, KEY, STATE, OUT> panes = createPanes(keySelector, function); panes.readFromInput(inView, keySerializer, stateTypeSerializer); restoredState = new RestoredState<>(panes, nextEvaluationTime, nextSlideTime); }
private void seekNextStateId(boolean metaFollows) throws Exception { int stateId = compressedInputView.readShort(); String stateName = stateIdMapping.get(stateId); if (metaFollows) { stateId = END_OF_KEY_GROUP_MARK & stateId; } while (!stateFilter.filter(stateName) && stateId != END_OF_KEY_GROUP_MARK) { final int keySize = compressedInputView.readInt(); final byte keyByte0 = compressedInputView.readByte(); compressedInputView.skip(keySize - 1); final int valueSize = compressedInputView.readInt(); compressedInputView.skip(valueSize); if (hasMetaDataFollowsFlag(keyByte0)) { stateId = END_OF_KEY_GROUP_MARK & compressedInputView.readShort(); stateName = stateIdMapping.get(stateId); } } this.stateId = stateId; this.stateName = stateName; }
public void clear() { this.buffer = initialBuffer; this.serializationReadBuffer.releaseArrays(); this.recordLength = -1; this.lengthBuffer.clear(); this.leftOverData = null; this.accumulatedRecordBytes = 0; if (spillingChannel != null) { try { spillingChannel.close(); } catch (Throwable t) { // ignore } spillingChannel = null; } if (spillFileReader != null) { try { spillFileReader.close(); } catch (Throwable t) { // ignore } spillFileReader = null; } if (spillFile != null) { spillFile.delete(); spillFile = null; } }
int sizeOfEventTimeTimers = stream.readInt(); if (sizeOfEventTimeTimers > 0) { Set<InternalTimer<K, N>> eventTimers = getEventTimeTimerSetForKeyGroup(keyGroupIdx); int sizeOfProcessingTimeTimers = stream.readInt(); if (sizeOfProcessingTimeTimers > 0) { Set<InternalTimer<K, N>> processingTimers = getProcessingTimeTimerSetForKeyGroup(keyGroupIdx);
private void restoreFromLegacyWindowOperator(DataInputViewStreamWrapper in) throws IOException { Preconditions.checkArgument(legacyWindowOperatorType == LegacyWindowOperatorType.NONE); int numWatermarkTimers = in.readInt(); this.restoredFromLegacyEventTimeTimers = new PriorityQueue<>(Math.max(numWatermarkTimers, 1)); K key = keySerializer.deserialize(in); W window = windowSerializer.deserialize(in); long timestamp = in.readLong(); int numProcessingTimeTimers = in.readInt(); this.restoredFromLegacyProcessingTimeTimers = new PriorityQueue<>(Math.max(numProcessingTimeTimers, 1)); K key = keySerializer.deserialize(in); W window = windowSerializer.deserialize(in); long timestamp = in.readLong(); int numProcessingTimeTimerTimestamp = in.readInt(); for (int i = 0; i < numProcessingTimeTimerTimestamp; i++) { in.readLong(); in.readInt();
public void restoreStateForKeyGroup(DataInputViewStreamWrapper stream, int keyGroupIdx, ClassLoader userCodeClassLoader) throws IOException, ClassNotFoundException { int noOfTimerServices = stream.readInt(); for (int i = 0; i < noOfTimerServices; i++) { String serviceName = stream.readUTF(); HeapInternalTimerService<K, N> timerService = timerServices.get(serviceName); if (timerService == null) { timerService = new HeapInternalTimerService<>( totalKeyGroups, localKeyGroupRange, keyContext, processingTimeService); timerServices.put(serviceName, timerService); } timerService.restoreTimersForKeyGroup(stream, keyGroupIdx, userCodeClassLoader); } }
private void restoreState() throws Exception { LOGGER.info("Restore siddhi state"); final Iterator<byte[]> siddhiState = siddhiRuntimeState.get().iterator(); if (siddhiState.hasNext()) { this.siddhiRuntime.restore(siddhiState.next()); } LOGGER.info("Restore queued records state"); final Iterator<byte[]> queueState = queuedRecordsState.get().iterator(); if (queueState.hasNext()) { final ByteArrayInputStream byteArrayInputStream = new ByteArrayInputStream(queueState.next()); final DataInputViewStreamWrapper dataInputView = new DataInputViewStreamWrapper(byteArrayInputStream); try { this.priorityQueue = restoreQueuerState(dataInputView); } finally { dataInputView.close(); byteArrayInputStream.close(); } } }
DataInputViewStreamWrapper inView = new DataInputViewStreamWrapper(fsDataInputStream); int writtenKeyGroupIndex = inView.readInt(); int kvStateId = inView.readShort(); StateTable<K, ?, ?> stateTable = stateTables.get(kvStatesById.get(kvStateId));
DataInputViewStreamWrapper inputView = new DataInputViewStreamWrapper(in); timerService = getInternalTimerService( "watermark-callbacks", int numberEntries = inputView.readInt(); for (int i = 0; i < numberEntries; i++) { KEY key = keySerializer.deserialize(inputView);
public void clear() { this.buffer = initialBuffer; this.serializationReadBuffer.releaseArrays(); this.recordLength = -1; this.lengthBuffer.clear(); this.leftOverData = null; this.accumulatedRecordBytes = 0; if (spillingChannel != null) { try { spillingChannel.close(); } catch (Throwable t) { // ignore } spillingChannel = null; } if (spillFileReader != null) { try { spillFileReader.close(); } catch (Throwable t) { // ignore } spillFileReader = null; } if (spillFile != null) { spillFile.delete(); spillFile = null; } }
private void readStateHandleStateData( FSDataInputStream fsDataInputStream, DataInputViewStreamWrapper inView, KeyGroupRangeOffsets keyGroupOffsets, Map<Integer, StateMetaInfoSnapshot> kvStatesById, int numStates, int readVersion, boolean isCompressed) throws IOException { final StreamCompressionDecorator streamCompressionDecorator = isCompressed ? SnappyStreamCompressionDecorator.INSTANCE : UncompressedStreamCompressionDecorator.INSTANCE; for (Tuple2<Integer, Long> groupOffset : keyGroupOffsets) { int keyGroupIndex = groupOffset.f0; long offset = groupOffset.f1; // Check that restored key groups all belong to the backend. Preconditions.checkState(keyGroupRange.contains(keyGroupIndex), "The key group must belong to the backend."); fsDataInputStream.seek(offset); int writtenKeyGroupIndex = inView.readInt(); Preconditions.checkState(writtenKeyGroupIndex == keyGroupIndex, "Unexpected key-group in restore."); try (InputStream kgCompressionInStream = streamCompressionDecorator.decorateWithCompression(fsDataInputStream)) { readKeyGroupStateData( kgCompressionInStream, kvStatesById, keyGroupIndex, numStates, readVersion); } } }