protected PriorityQueueSetFactory createQueueFactory(KeyGroupRange keyGroupRange, int numKeyGroups) { return new HeapPriorityQueueSetFactory(keyGroupRange, numKeyGroups, 128); }
public int numStateEntries(W window) { return stateBackend.numKeyValueStateEntries(window); }
/** * Tests list serialization utils. */ @Test public void testListSerialization() throws Exception { final long key = 0L; final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0); // objects for heap state list serialisation final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend = new HeapKeyedStateBackend<>( mock(TaskKvStateRegistry.class), LongSerializer.INSTANCE, ClassLoader.getSystemClassLoader(), keyGroupRange.getNumberOfKeyGroups(), keyGroupRange, async, new ExecutionConfig(), TestLocalRecoveryConfig.disabled(), new HeapPriorityQueueSetFactory(keyGroupRange, keyGroupRange.getNumberOfKeyGroups(), 128), TtlTimeProvider.DEFAULT ); longHeapKeyedStateBackend.setCurrentKey(key); final InternalListState<Long, VoidNamespace, Long> listState = longHeapKeyedStateBackend.createInternalState( VoidNamespaceSerializer.INSTANCE, new ListStateDescriptor<>("test", LongSerializer.INSTANCE)); testListSerialization(key, listState); }
/** * Tests map serialization utils. */ @Test public void testMapSerialization() throws Exception { final long key = 0L; final KeyGroupRange keyGroupRange = new KeyGroupRange(0, 0); // objects for heap state list serialisation final HeapKeyedStateBackend<Long> longHeapKeyedStateBackend = new HeapKeyedStateBackend<>( mock(TaskKvStateRegistry.class), LongSerializer.INSTANCE, ClassLoader.getSystemClassLoader(), keyGroupRange.getNumberOfKeyGroups(), keyGroupRange, async, new ExecutionConfig(), TestLocalRecoveryConfig.disabled(), new HeapPriorityQueueSetFactory(keyGroupRange, keyGroupRange.getNumberOfKeyGroups(), 128), TtlTimeProvider.DEFAULT ); longHeapKeyedStateBackend.setCurrentKey(key); final InternalMapState<Long, VoidNamespace, Long, String> mapState = (InternalMapState<Long, VoidNamespace, Long, String>) longHeapKeyedStateBackend.getPartitionedState( VoidNamespace.INSTANCE, VoidNamespaceSerializer.INSTANCE, new MapStateDescriptor<>("test", LongSerializer.INSTANCE, StringSerializer.INSTANCE)); testMapSerialization(key, mapState); }
@Override public V value() { final V result = stateTable.get(currentNamespace); if (result == null) { return getDefaultValue(); } return result; }
@Override public <N, T, ACC, R> InternalAggregatingState<N, T, R> createAggregatingState( TypeSerializer<N> namespaceSerializer, AggregatingStateDescriptor<T, ACC, R> stateDesc) throws Exception { StateTable<K, N, ACC> stateTable = tryRegisterStateTable(namespaceSerializer, stateDesc); return new HeapAggregatingState<>(stateDesc, stateTable, keySerializer, namespaceSerializer); }
@Override public <N, T> InternalReducingState<N, T> createReducingState( TypeSerializer<N> namespaceSerializer, ReducingStateDescriptor<T> stateDesc) throws Exception { StateTable<K, N, T> stateTable = tryRegisterStateTable(namespaceSerializer, stateDesc); return new HeapReducingState<>(stateDesc, stateTable, keySerializer, namespaceSerializer); }
@Override public <N, T, ACC> InternalFoldingState<N, T, ACC> createFoldingState( TypeSerializer<N> namespaceSerializer, FoldingStateDescriptor<T, ACC> stateDesc) throws Exception { StateTable<K, N, ACC> stateTable = tryRegisterStateTable(namespaceSerializer, stateDesc); return new HeapFoldingState<>(stateDesc, stateTable, keySerializer, namespaceSerializer); }
public TriggerTestHarness( Trigger<T, W> trigger, TypeSerializer<W> windowSerializer) throws Exception { this.trigger = trigger; this.windowSerializer = windowSerializer; // we only ever use one key, other tests make sure that windows work across different // keys DummyEnvironment dummyEnv = new DummyEnvironment("test", 1, 0); MemoryStateBackend backend = new MemoryStateBackend(); @SuppressWarnings("unchecked") HeapKeyedStateBackend<Integer> stateBackend = (HeapKeyedStateBackend<Integer>) backend.createKeyedStateBackend(dummyEnv, new JobID(), "test_op", IntSerializer.INSTANCE, 1, new KeyGroupRange(0, 0), new KvStateRegistry().createTaskRegistry(new JobID(), new JobVertexID())); this.stateBackend = stateBackend; this.stateBackend.setCurrentKey(KEY); this.internalTimerService = new TestInternalTimerService<>(new KeyContext() { @Override public void setCurrentKey(Object key) { // ignore } @Override public Object getCurrentKey() { return KEY; } }); }
@Override int tryAddToSource(int currentIndex, CopyOnWriteStateTable.StateTableEntry<K, N, S> entry) { CopyOnWriteStateTable.StateTableEntry<K, N, S> filteredEntry = filterEntry(entry); if (filteredEntry != null) { return super.tryAddToSource(currentIndex, filteredEntry); } return currentIndex; }
/** * This tests that {@link Lockable.LockableTypeSerializer#duplicate()} works as expected. */ @Test public void testDuplicate() { IntSerializer nonDuplicatingInnerSerializer = IntSerializer.INSTANCE; Assert.assertSame(nonDuplicatingInnerSerializer, nonDuplicatingInnerSerializer.duplicate()); Lockable.LockableTypeSerializer<Integer> candidateTestShallowDuplicate = new Lockable.LockableTypeSerializer<>(nonDuplicatingInnerSerializer); Assert.assertSame(candidateTestShallowDuplicate, candidateTestShallowDuplicate.duplicate()); TestDuplicateSerializer duplicatingInnerSerializer = new TestDuplicateSerializer(); Assert.assertNotSame(duplicatingInnerSerializer, duplicatingInnerSerializer.duplicate()); Lockable.LockableTypeSerializer<Integer> candidateTestDeepDuplicate = new Lockable.LockableTypeSerializer<>(duplicatingInnerSerializer); Lockable.LockableTypeSerializer<Integer> deepDuplicate = candidateTestDeepDuplicate.duplicate(); Assert.assertNotSame(candidateTestDeepDuplicate, deepDuplicate); Assert.assertNotSame(candidateTestDeepDuplicate.getElementSerializer(), deepDuplicate.getElementSerializer()); } }
public <N, V> StateTable<K, N, V> newStateTable(RegisteredKeyedBackendStateMetaInfo<N, V> newMetaInfo) { return asynchronousSnapshots ? new CopyOnWriteStateTable<>(this, newMetaInfo) : new NestedMapsStateTable<>(this, newMetaInfo); }
this.priorityQueueFactory = new HeapPriorityQueueSetFactory(keyGroupRange, numberOfKeyGroups, 128); break; case ROCKSDB:
@Override public V value() { final V result = stateTable.get(currentNamespace); if (result == null) { return getDefaultValue(); } return result; }
public int numStateEntries() { return stateBackend.numKeyValueStateEntries(); }
new HeapPriorityQueueSetFactory(testKeyGroupRange, maxParallelism, 128); InternalTimerServiceImpl<Integer, String> timerService = createAndStartInternalTimerService(mockTriggerable, keyContext, processingTimeService, testKeyGroupRange, priorityQueueSetFactory);
public <N> int numKeyedStateEntries(N namespace) { AbstractStreamOperator<?> abstractStreamOperator = (AbstractStreamOperator<?>) operator; KeyedStateBackend<Object> keyedStateBackend = abstractStreamOperator.getKeyedStateBackend(); if (keyedStateBackend instanceof HeapKeyedStateBackend) { return ((HeapKeyedStateBackend) keyedStateBackend).numKeyValueStateEntries(namespace); } else { throw new UnsupportedOperationException(); } } }
public int numKeyedStateEntries() { AbstractStreamOperator<?> abstractStreamOperator = (AbstractStreamOperator<?>) operator; KeyedStateBackend<Object> keyedStateBackend = abstractStreamOperator.getKeyedStateBackend(); if (keyedStateBackend instanceof HeapKeyedStateBackend) { return ((HeapKeyedStateBackend) keyedStateBackend).numKeyValueStateEntries(); } else { throw new UnsupportedOperationException(); } } }
public int numKeyedStateEntries() { AbstractStreamOperator<?> abstractStreamOperator = (AbstractStreamOperator<?>) operator; KeyedStateBackend<Object> keyedStateBackend = abstractStreamOperator.getKeyedStateBackend(); if (keyedStateBackend instanceof HeapKeyedStateBackend) { return ((HeapKeyedStateBackend) keyedStateBackend).numKeyValueStateEntries(); } else { throw new UnsupportedOperationException(); } }