/** * Acquires a releaseable lock for the given uId. All *UnderLock methods require * this lock to be hold by the caller otherwise the visibility guarantees of this version * map are broken. We assert on this lock to be hold when calling these methods. * @see KeyedLock */ Releasable acquireLock(BytesRef uid) { return keyedLock.acquire(uid); }
boolean assertKeyedLockHeldByCurrentThread(BytesRef uid) { assert keyedLock.isHeldByCurrentThread(uid) : "Thread [" + Thread.currentThread().getName() + "], uid [" + uid.utf8ToString() + "]"; return true; } }
/** * Try to prune tombstones whose timestamp is less than maxTimestampToPrune and seqno at most the maxSeqNoToPrune. */ void pruneTombstones(long maxTimestampToPrune, long maxSeqNoToPrune) { for (Map.Entry<BytesRef, DeleteVersionValue> entry : tombstones.entrySet()) { // we do check before we actually lock the key - this way we don't need to acquire the lock for tombstones that are not // prune-able. If the tombstone changes concurrently we will re-read and step out below since if we can't collect it now w // we won't collect the tombstone below since it must be newer than this one. if (canRemoveTombstone(maxTimestampToPrune, maxSeqNoToPrune, entry.getValue())) { final BytesRef uid = entry.getKey(); try (Releasable lock = keyedLock.tryAcquire(uid)) { // we use tryAcquire here since this is a best effort and we try to be least disruptive // this method is also called under lock in the engine under certain situations such that this can lead to deadlocks // if we do use a blocking acquire. see #28714 if (lock != null) { // did we get the lock? // Must re-get it here, vs using entry.getValue(), in case the uid was indexed/deleted since we pulled the iterator: final DeleteVersionValue versionValue = tombstones.get(uid); if (versionValue != null) { if (canRemoveTombstone(maxTimestampToPrune, maxSeqNoToPrune, versionValue)) { removeTombstoneUnderLock(uid); } } } } } } }
@Override public void disconnectFromNode(DiscoveryNode node) { NodeChannels nodeChannels = connectedNodes.remove(node); if (nodeChannels != null) { connectionLock.acquire(node.id()); try { try { nodeChannels.close(); } finally { logger.debug("disconnected from [{}]", node); transportServiceAdapter.raiseNodeDisconnected(node); } } finally { connectionLock.release(node.id()); } } }
/** * Acquires a lock for the given key. The key is compared by it's equals method not by object identity. The lock can be acquired * by the same thread multiple times. The lock is released by closing the returned {@link Releasable}. */ public Releasable acquire(T key) { while (true) { KeyLock perNodeLock = map.get(key); if (perNodeLock == null) { ReleasableLock newLock = tryCreateNewLock(key); if (newLock != null) { return newLock; } } else { assert perNodeLock != null; int i = perNodeLock.count.get(); if (i > 0 && perNodeLock.count.compareAndSet(i, i + 1)) { perNodeLock.lock(); return new ReleasableLock(key, perNodeLock); } } } }
private final KeyedLock<BytesRef> keyedLock = new KeyedLock<>();
/** * Disconnects from a node if a channel is found as part of that nodes channels. */ private void disconnectFromNodeChannel(Channel channel, Throwable failure) { for (DiscoveryNode node : connectedNodes.keySet()) { NodeChannels nodeChannels = connectedNodes.get(node); if (nodeChannels != null && nodeChannels.hasChannel(channel)) { connectionLock.acquire(node.id()); if (!nodeChannels.hasChannel(channel)) { //might have been removed in the meanwhile, safety check assert !connectedNodes.containsKey(node); } else { try { connectedNodes.remove(node); try { nodeChannels.close(); } finally { logger.debug("disconnected from [{}] on channel failure", failure, node); transportServiceAdapter.raiseNodeDisconnected(node); } } finally { connectionLock.release(node.id()); } } } } }
/** * Tries to acquire the lock for the given key and returns it. If the lock can't be acquired null is returned. */ public Releasable tryAcquire(T key) { final KeyLock perNodeLock = map.get(key); if (perNodeLock == null) { return tryCreateNewLock(key); } if (perNodeLock.tryLock()) { // ok we got it - make sure we increment it accordingly otherwise release it again int i; while ((i = perNodeLock.count.get()) > 0) { // we have to do this in a loop here since even if the count is > 0 // there could be a concurrent blocking acquire that changes the count and then this CAS fails. Since we already got // the lock we should retry and see if we can still get it or if the count is 0. If that is the case and we give up. if (perNodeLock.count.compareAndSet(i, i + 1)) { return new ReleasableLock(key, perNodeLock); } } perNodeLock.unlock(); // make sure we unlock and don't leave the lock in a locked state } return null; }
protected void doRun() { for (DiscoveryNode node : nodes.keySet()) { try (Releasable ignored = nodeLocks.acquire(node)) { validateAndConnectIfNeeded(node); } } }
void validateAndConnectIfNeeded(DiscoveryNode node) { assert nodeLocks.isHeldByCurrentThread(node) : "validateAndConnectIfNeeded must be called under lock"; if (lifecycle.stoppedOrClosed() || nodes.containsKey(node) == false) { // we double check existence of node since connectToNode might take time... // nothing to do } else { try { // connecting to an already connected node is a noop transportService.connectToNode(node); nodes.put(node, 0); } catch (Exception e) { Integer nodeFailureCount = nodes.get(node); assert nodeFailureCount != null : node + " didn't have a counter in nodes map"; nodeFailureCount = nodeFailureCount + 1; // log every 6th failure if ((nodeFailureCount % 6) == 1) { final int finalNodeFailureCount = nodeFailureCount; logger.warn(() -> new ParameterizedMessage( "failed to connect to node {} (tried [{}] times)", node, finalNodeFailureCount), e); } nodes.put(node, nodeFailureCount); } } }
/** * Acquires a lock for the given key. The key is compared by it's equals method not by object identity. The lock can be acquired * by the same thread multiple times. The lock is released by closing the returned {@link Releasable}. */ public Releasable acquire(T key) { while (true) { KeyLock perNodeLock = map.get(key); if (perNodeLock == null) { ReleasableLock newLock = tryCreateNewLock(key); if (newLock != null) { return newLock; } } else { assert perNodeLock != null; int i = perNodeLock.count.get(); if (i > 0 && perNodeLock.count.compareAndSet(i, i + 1)) { perNodeLock.lock(); return new ReleasableLock(key, perNodeLock); } } } }
/** * Try to prune tombstones whose timestamp is less than maxTimestampToPrune and seqno at most the maxSeqNoToPrune. */ void pruneTombstones(long maxTimestampToPrune, long maxSeqNoToPrune) { for (Map.Entry<BytesRef, DeleteVersionValue> entry : tombstones.entrySet()) { // we do check before we actually lock the key - this way we don't need to acquire the lock for tombstones that are not // prune-able. If the tombstone changes concurrently we will re-read and step out below since if we can't collect it now w // we won't collect the tombstone below since it must be newer than this one. if (canRemoveTombstone(maxTimestampToPrune, maxSeqNoToPrune, entry.getValue())) { final BytesRef uid = entry.getKey(); try (Releasable lock = keyedLock.tryAcquire(uid)) { // we use tryAcquire here since this is a best effort and we try to be least disruptive // this method is also called under lock in the engine under certain situations such that this can lead to deadlocks // if we do use a blocking acquire. see #28714 if (lock != null) { // did we get the lock? // Must re-get it here, vs using entry.getValue(), in case the uid was indexed/deleted since we pulled the iterator: final DeleteVersionValue versionValue = tombstones.get(uid); if (versionValue != null) { if (canRemoveTombstone(maxTimestampToPrune, maxSeqNoToPrune, versionValue)) { removeTombstoneUnderLock(uid); } } } } } } }
@Override protected void doRun() { try (Releasable ignored = nodeLocks.acquire(node)) { validateAndConnectIfNeeded(node); } }
/** * Checks if the given operation has been processed in this engine or not. * @return true if the given operation was processed; otherwise false. */ protected final boolean hasBeenProcessedBefore(Operation op) { if (Assertions.ENABLED) { assert op.seqNo() != SequenceNumbers.UNASSIGNED_SEQ_NO : "operation is not assigned seq_no"; if (op.operationType() == Operation.TYPE.NO_OP) { assert noOpKeyedLock.isHeldByCurrentThread(op.seqNo()); } else { assert versionMap.assertKeyedLockHeldByCurrentThread(op.uid().bytes()); } } return localCheckpointTracker.contains(op.seqNo()); }
/** * Acquires a lock for the given key. The key is compared by it's equals method not by object identity. The lock can be acquired * by the same thread multiple times. The lock is released by closing the returned {@link Releasable}. */ public Releasable acquire(T key) { while (true) { KeyLock perNodeLock = map.get(key); if (perNodeLock == null) { ReleasableLock newLock = tryCreateNewLock(key); if (newLock != null) { return newLock; } } else { assert perNodeLock != null; int i = perNodeLock.count.get(); if (i > 0 && perNodeLock.count.compareAndSet(i, i + 1)) { perNodeLock.lock(); return new ReleasableLock(key, perNodeLock); } } } }
/** * Try to prune tombstones whose timestamp is less than maxTimestampToPrune and seqno at most the maxSeqNoToPrune. */ void pruneTombstones(long maxTimestampToPrune, long maxSeqNoToPrune) { for (Map.Entry<BytesRef, DeleteVersionValue> entry : tombstones.entrySet()) { // we do check before we actually lock the key - this way we don't need to acquire the lock for tombstones that are not // prune-able. If the tombstone changes concurrently we will re-read and step out below since if we can't collect it now w // we won't collect the tombstone below since it must be newer than this one. if (canRemoveTombstone(maxTimestampToPrune, maxSeqNoToPrune, entry.getValue())) { final BytesRef uid = entry.getKey(); try (Releasable lock = keyedLock.tryAcquire(uid)) { // we use tryAcquire here since this is a best effort and we try to be least disruptive // this method is also called under lock in the engine under certain situations such that this can lead to deadlocks // if we do use a blocking acquire. see #28714 if (lock != null) { // did we get the lock? // Must re-get it here, vs using entry.getValue(), in case the uid was indexed/deleted since we pulled the iterator: final DeleteVersionValue versionValue = tombstones.get(uid); if (versionValue != null) { if (canRemoveTombstone(maxTimestampToPrune, maxSeqNoToPrune, versionValue)) { removeTombstoneUnderLock(uid); } } } } } } }
/** * Disconnects from all nodes except the ones provided as parameter */ public void disconnectFromNodesExcept(DiscoveryNodes nodesToKeep) { Set<DiscoveryNode> currentNodes = new HashSet<>(nodes.keySet()); for (DiscoveryNode node : nodesToKeep) { currentNodes.remove(node); } for (final DiscoveryNode node : currentNodes) { try (Releasable ignored = nodeLocks.acquire(node)) { Integer current = nodes.remove(node); assert current != null : "node " + node + " was removed in event but not in internal nodes"; try { transportService.disconnectFromNode(node); } catch (Exception e) { logger.warn(() -> new ParameterizedMessage("failed to disconnect to node [{}]", node), e); } } } }
private boolean assertKeyedLockHeldByCurrentThread(BytesRef uid) { assert keyedLock.isHeldByCurrentThread(uid) : "Thread [" + Thread.currentThread().getName() + "], uid [" + uid.utf8ToString() + "]"; return true; } }
/** * Tries to acquire the lock for the given key and returns it. If the lock can't be acquired null is returned. */ public Releasable tryAcquire(T key) { final KeyLock perNodeLock = map.get(key); if (perNodeLock == null) { return tryCreateNewLock(key); } if (perNodeLock.tryLock()) { // ok we got it - make sure we increment it accordingly otherwise release it again int i; while ((i = perNodeLock.count.get()) > 0) { // we have to do this in a loop here since even if the count is > 0 // there could be a concurrent blocking acquire that changes the count and then this CAS fails. Since we already got // the lock we should retry and see if we can still get it or if the count is 0. If that is the case and we give up. if (perNodeLock.count.compareAndSet(i, i + 1)) { return new ReleasableLock(key, perNodeLock); } } perNodeLock.unlock(); // make sure we unlock and don't leave the lock in a locked state } return null; }