/** * Cancels the recovery and interrupts all eligible threads. */ public void cancel(String reason) { cancellableThreads.cancel(reason); }
boolean wasInterrupted = add(); boolean cancelledByExternalInterrupt = false; RuntimeException runtimeException = null; ioException = e; } finally { remove(); if (isCancelled()) { onCancel(reason, ioException != null ? ioException : runtimeException); } else if (ioException != null) {
throw new IndexShardClosedException(request.shardId()); cancellableThreads.checkForCancel(); StopWatch stopWatch = new StopWatch().start(); logger.trace("finalizing recovery"); shardId + " marking " + request.targetAllocationId() + " as in sync", shard, cancellableThreads, logger); final long globalCheckpoint = shard.getGlobalCheckpoint(); cancellableThreads.executeIO(() -> recoveryTarget.finalizeRecovery(globalCheckpoint)); runUnderPrimaryPermit(() -> shard.updateGlobalCheckpointForShard(request.targetAllocationId(), globalCheckpoint), shardId + " updating " + request.targetAllocationId() + "'s global checkpoint", shard, cancellableThreads, logger); logger.trace("performing relocation hand-off"); cancellableThreads.execute(() -> shard.relocated(recoveryTarget::handoffPrimaryContext));
/** call this will throw an exception if operation was cancelled. * Override {@link #onCancel(String, Exception)} for custom failure logic */ public synchronized void checkForCancel() { if (isCancelled()) { onCancel(reason, null); } }
throw new IndexShardClosedException(request.shardId()); cancellableThreads.checkForCancel(); cancellableThreads.executeIO(sendBatch); logger.trace("sent batch of [{}][{}] (total: [{}]) translog operations", ops, new ByteSizeValue(size), expectedTotalOps); ops = 0; cancellableThreads.executeIO(sendBatch);
throw new IndexShardClosedException(request.shardId()); cancellableThreads.checkForCancel(); StopWatch stopWatch = new StopWatch().start(); logger.trace("[{}][{}] finalizing recovery to {}", indexName, shardId, request.targetNode()); cancellableThreads.execute(recoveryTarget::finalizeRecovery); logger.trace("[{}][{}] waiting on {} to have cluster state with version [{}]", indexName, shardId, request.targetNode(), currentClusterStateVersion); cancellableThreads.execute(() -> recoveryTarget.ensureClusterStateVersion(currentClusterStateVersion)); cancellableThreads.execute(() -> shard.relocated("to " + request.targetNode()));
/** * run the Interruptable, capturing the executing thread. Concurrent calls to {@link #cancel(String)} will interrupt this thread * causing the call to prematurely return. * * @param interruptable code to run */ public void execute(Interruptable interruptable) { try { executeIO(interruptable); } catch (IOException e) { assert false : "the passed interruptable can not result in an IOException"; throw new RuntimeException("unexpected IO exception", e); } } /**
newTargetCancellableThreads.execute(closedLatch::await); } catch (CancellableThreads.ExecutionCancelledException e) { logger.trace("new recovery target cancelled for shard {} while waiting on old recovery target with id [{}] to close",
private synchronized boolean add() { checkForCancel(); threads.add(Thread.currentThread()); // capture and clean the interrupted thread before we start, so we can identify // our own interrupt. we do so under lock so we know we don't clear our own. return Thread.interrupted(); }
/** * Creates a new recovery target object that represents a recovery to the provided shard. * * @param indexShard local shard where we want to recover to * @param sourceNode source node of the recovery where we recover from * @param listener called when recovery is completed/failed * @param ensureClusterStateVersionCallback callback to ensure that the current node is at least on a cluster state with the provided * version; necessary for primary relocation so that new primary knows about all other ongoing * replica recoveries when replicating documents (see {@link RecoverySourceHandler}) */ public RecoveryTarget(final IndexShard indexShard, final DiscoveryNode sourceNode, final PeerRecoveryTargetService.RecoveryListener listener, final LongConsumer ensureClusterStateVersionCallback) { super("recovery_status"); this.cancellableThreads = new CancellableThreads(); this.recoveryId = idGenerator.incrementAndGet(); this.listener = listener; this.logger = Loggers.getLogger(getClass(), indexShard.shardId()); this.indexShard = indexShard; this.sourceNode = sourceNode; this.shardId = indexShard.shardId(); this.tempFilePrefix = RECOVERY_PREFIX + UUIDs.randomBase64UUID() + "."; this.store = indexShard.store(); this.ensureClusterStateVersionCallback = ensureClusterStateVersionCallback; // make sure the store is not released until we are done. store.incRef(); indexShard.recoveryStats().incCurrentAsTarget(); }
throw new IndexShardClosedException(request.shardId()); cancellableThreads.checkForCancel(); operations.add(operation); ops += 1; cancellableThreads.execute(() -> recoveryTarget.indexTranslogOperations(operations, snapshot.totalOperations())); if (logger.isTraceEnabled()) { logger.trace("[{}][{}] sent batch of [{}][{}] (total: [{}]) translog operations to {}", cancellableThreads.execute(() -> recoveryTarget.indexTranslogOperations(operations, snapshot.totalOperations()));
private void sendNextChunk(long position, BytesArray content, boolean lastChunk) throws IOException { // Actually send the file chunk to the target node, waiting for it to complete cancellableThreads.executeIO(() -> recoveryTarget.writeFileChunk(md, position, content, lastChunk, translogOps.get()) ); if (shard.state() == IndexShardState.CLOSED) { // check if the shard got closed on us throw new IndexShardClosedException(request.shardId()); } } }
/** call this will throw an exception if operation was cancelled. Override {@link #onCancel(String, Exception)} for custom failure logic */ public synchronized void checkForCancel() { if (isCancelled()) { onCancel(reason, null); } }
static void runUnderPrimaryPermit(CancellableThreads.Interruptable runnable, String reason, IndexShard primary, CancellableThreads cancellableThreads, Logger logger) { cancellableThreads.execute(() -> { CompletableFuture<Releasable> permit = new CompletableFuture<>(); final ActionListener<Releasable> onAcquired = new ActionListener<Releasable>() {
throw new IndexShardClosedException(request.shardId()); cancellableThreads.checkForCancel();
throw new IndexShardClosedException(request.shardId()); cancellableThreads.checkForCancel(); cancellableThreads.executeIO(sendBatch); logger.trace("sent batch of [{}][{}] (total: [{}]) translog operations", ops, new ByteSizeValue(size), expectedTotalOps); ops = 0; cancellableThreads.executeIO(sendBatch);
private final AtomicBoolean closed = new AtomicBoolean(false); private final BlockingQueue<ActionListener<Void>> queue = new ArrayBlockingQueue<>(100); private final CancellableThreads cancellableThreads = new CancellableThreads();
boolean wasInterrupted = add(); RuntimeException throwable = null; try { throwable = t; } finally { remove(); if (isCancelled()) { onCancel(reason, throwable); } else if (throwable != null) {
cancellableThreads.checkForCancel(); response.phase1FileNames.size(), new ByteSizeValue(totalSize), response.phase1ExistingFileNames.size(), new ByteSizeValue(existingTotalSize)); cancellableThreads.execute(() -> recoveryTarget.receiveFileInfo(response.phase1FileNames, response.phase1FileSizes, response.phase1ExistingFileNames, response.phase1ExistingFileSizes, translogOps.get())); cancellableThreads.executeIO(() -> recoveryTarget.cleanFiles(translogOps.get(), recoverySourceMetadata)); } catch (RemoteTransportException | IOException targetException) { cancellableThreads.checkForCancel(); logger.debug("checking integrity for file {} after remove corruption exception", md); if (store.checkIntegrityNoException(md) == false) { // we are corrupted on the primary -- fail!
@Override public void close() throws IOException { try { if (closed.compareAndSet(false, true)) { cancellableThreads.cancel("connect handler is closed"); running.acquire(); // acquire the semaphore to ensure all connections are closed and all thread joined running.release(); maybeConnect(); // now go and notify pending listeners } } catch (InterruptedException e) { Thread.currentThread().interrupt(); } }