/** * Allocate a table of the given capacity and set the threshold accordingly. * * @param newCapacity must be a power of two */ private StateTableEntry<K, N, S>[] makeTable(int newCapacity) { if (newCapacity < MAXIMUM_CAPACITY) { threshold = (newCapacity >> 1) + (newCapacity >> 2); // 3/4 capacity } else { if (size() > MAX_ARRAY_SIZE) { throw new IllegalStateException("Maximum capacity of CopyOnWriteStateTable is reached and the job " + "cannot continue. Please consider scaling-out your job or using a different keyed state backend " + "implementation!"); } else { LOG.warn("Maximum capacity of 2^30 in StateTable reached. Cannot increase hash table size. This can " + "lead to more collisions and lower performance. Please consider scaling-out your job or using a " + "different keyed state backend implementation!"); threshold = MAX_ARRAY_SIZE; } } @SuppressWarnings("unchecked") StateTableEntry<K, N, S>[] newTable = (StateTableEntry<K, N, S>[]) new StateTableEntry[newCapacity]; return newTable; }
/** * Allocate a table of the given capacity and set the threshold accordingly. * * @param newCapacity must be a power of two */ private StateTableEntry<K, N, S>[] makeTable(int newCapacity) { if (newCapacity < MAXIMUM_CAPACITY) { threshold = (newCapacity >> 1) + (newCapacity >> 2); // 3/4 capacity } else { if (size() > MAX_ARRAY_SIZE) { throw new IllegalStateException("Maximum capacity of CopyOnWriteStateTable is reached and the job " + "cannot continue. Please consider scaling-out your job or using a different keyed state backend " + "implementation!"); } else { LOG.warn("Maximum capacity of 2^30 in StateTable reached. Cannot increase hash table size. This can " + "lead to more collisions and lower performance. Please consider scaling-out your job or using a " + "different keyed state backend implementation!"); threshold = MAX_ARRAY_SIZE; } } @SuppressWarnings("unchecked") StateTableEntry<K, N, S>[] newTable = (StateTableEntry<K, N, S>[]) new StateTableEntry[newCapacity]; return newTable; }
final int copiedArraySize = Math.max(totalTableIndexSize, size()); final StateTableEntry<K, N, S>[] copy = new StateTableEntry[copiedArraySize];
final int copiedArraySize = Math.max(totalTableIndexSize, size()); final StateTableEntry<K, N, S>[] copy = new StateTableEntry[copiedArraySize];
/** * Helper method that is the basis for operations that add mappings. */ private StateTableEntry<K, N, S> putEntry(K key, N namespace) { final int hash = computeHashForOperationAndDoIncrementalRehash(key, namespace); final StateTableEntry<K, N, S>[] tab = selectActiveTable(hash); int index = hash & (tab.length - 1); for (StateTableEntry<K, N, S> e = tab[index]; e != null; e = e.next) { if (e.hash == hash && key.equals(e.key) && namespace.equals(e.namespace)) { // copy-on-write check for entry if (e.entryVersion < highestRequiredSnapshotVersion) { e = handleChainedEntryCopyOnWrite(tab, index, e); } return e; } } ++modCount; if (size() > threshold) { doubleCapacity(); } return addNewStateTableEntry(tab, key, namespace, hash); }
/** * Helper method that is the basis for operations that add mappings. */ private StateTableEntry<K, N, S> putEntry(K key, N namespace) { final int hash = computeHashForOperationAndDoIncrementalRehash(key, namespace); final StateTableEntry<K, N, S>[] tab = selectActiveTable(hash); int index = hash & (tab.length - 1); for (StateTableEntry<K, N, S> e = tab[index]; e != null; e = e.next) { if (e.hash == hash && key.equals(e.key) && namespace.equals(e.namespace)) { // copy-on-write check for entry if (e.entryVersion < highestRequiredSnapshotVersion) { e = handleChainedEntryCopyOnWrite(tab, index, e); } return e; } } ++modCount; if (size() > threshold) { doubleCapacity(); } return addNewStateTableEntry(tab, key, namespace, hash); }
/** * Helper method that is the basis for operations that add mappings. */ private StateTableEntry<K, N, S> putEntry(K key, N namespace) { final int hash = computeHashForOperationAndDoIncrementalRehash(key, namespace); final StateTableEntry<K, N, S>[] tab = selectActiveTable(hash); int index = hash & (tab.length - 1); for (StateTableEntry<K, N, S> e = tab[index]; e != null; e = e.next) { if (e.hash == hash && key.equals(e.key) && namespace.equals(e.namespace)) { // copy-on-write check for entry if (e.entryVersion < highestRequiredSnapshotVersion) { e = handleChainedEntryCopyOnWrite(tab, index, e); } return e; } } ++modCount; if (size() > threshold) { doubleCapacity(); } return addNewStateTableEntry(tab, key, namespace, hash); }
/** * Helper method that is the basis for operations that add mappings. */ private StateTableEntry<K, N, S> putEntry(K key, N namespace) { final int hash = computeHashForOperationAndDoIncrementalRehash(key, namespace); final StateTableEntry<K, N, S>[] tab = selectActiveTable(hash); int index = hash & (tab.length - 1); for (StateTableEntry<K, N, S> e = tab[index]; e != null; e = e.next) { if (e.hash == hash && key.equals(e.key) && namespace.equals(e.namespace)) { // copy-on-write check for entry if (e.entryVersion < highestRequiredSnapshotVersion) { e = handleChainedEntryCopyOnWrite(tab, index, e); } return e; } } ++modCount; if (size() > threshold) { doubleCapacity(); } return addNewStateTableEntry(tab, key, namespace, hash); }
/** * Creates a new {@link CopyOnWriteStateTableSnapshot}. * * @param owningStateTable the {@link CopyOnWriteStateTable} for which this object represents a snapshot. */ CopyOnWriteStateTableSnapshot(CopyOnWriteStateTable<K, N, S> owningStateTable) { super(owningStateTable); this.snapshotData = owningStateTable.snapshotTableArrays(); this.snapshotVersion = owningStateTable.getStateTableVersion(); this.numberOfEntriesInSnapshotData = owningStateTable.size(); // We create duplicates of the serializers for the async snapshot, because TypeSerializer // might be stateful and shared with the event processing thread. this.localKeySerializer = owningStateTable.keyContext.getKeySerializer().duplicate(); this.localNamespaceSerializer = owningStateTable.metaInfo.getNamespaceSerializer().duplicate(); this.localStateSerializer = owningStateTable.metaInfo.getStateSerializer().duplicate(); this.partitionedStateTableSnapshot = null; }
/** * Creates a new {@link CopyOnWriteStateTableSnapshot}. * * @param owningStateTable the {@link CopyOnWriteStateTable} for which this object represents a snapshot. */ CopyOnWriteStateTableSnapshot(CopyOnWriteStateTable<K, N, S> owningStateTable) { super(owningStateTable); this.snapshotData = owningStateTable.snapshotTableArrays(); this.snapshotVersion = owningStateTable.getStateTableVersion(); this.stateTableSize = owningStateTable.size(); // We create duplicates of the serializers for the async snapshot, because TypeSerializer // might be stateful and shared with the event processing thread. this.localKeySerializer = owningStateTable.keyContext.getKeySerializer().duplicate(); this.localNamespaceSerializer = owningStateTable.metaInfo.getNamespaceSerializer().duplicate(); this.localStateSerializer = owningStateTable.metaInfo.getStateSerializer().duplicate(); this.keyGroupOffsets = null; }
/** * Creates a new {@link CopyOnWriteStateTableSnapshot}. * * @param owningStateTable the {@link CopyOnWriteStateTable} for which this object represents a snapshot. */ CopyOnWriteStateTableSnapshot(CopyOnWriteStateTable<K, N, S> owningStateTable) { super(owningStateTable); this.snapshotData = owningStateTable.snapshotTableArrays(); this.snapshotVersion = owningStateTable.getStateTableVersion(); this.stateTableSize = owningStateTable.size(); // We create duplicates of the serializers for the async snapshot, because TypeSerializer // might be stateful and shared with the event processing thread. this.localKeySerializer = owningStateTable.keyContext.getKeySerializer().duplicate(); this.localNamespaceSerializer = owningStateTable.metaInfo.getNamespaceSerializer().duplicate(); this.localStateSerializer = owningStateTable.metaInfo.getStateSerializer().duplicate(); this.keyGroupOffsets = null; }
/** * Creates a new {@link CopyOnWriteStateTableSnapshot}. * * @param owningStateTable the {@link CopyOnWriteStateTable} for which this object represents a snapshot. */ CopyOnWriteStateTableSnapshot(CopyOnWriteStateTable<K, N, S> owningStateTable) { super(owningStateTable); this.snapshotData = owningStateTable.snapshotTableArrays(); this.snapshotVersion = owningStateTable.getStateTableVersion(); this.numberOfEntriesInSnapshotData = owningStateTable.size(); // We create duplicates of the serializers for the async snapshot, because TypeSerializer // might be stateful and shared with the event processing thread. this.localKeySerializer = owningStateTable.keyContext.getKeySerializer().duplicate(); this.localNamespaceSerializer = owningStateTable.metaInfo.getNamespaceSerializer().duplicate(); this.localStateSerializer = owningStateTable.metaInfo.getStateSerializer().duplicate(); this.partitionedStateTableSnapshot = null; }