@Override protected ShardSegments shardOperation(IndicesSegmentsRequest request, ShardRouting shardRouting) { IndexService indexService = indicesService.indexServiceSafe(shardRouting.index()); IndexShard indexShard = indexService.getShard(shardRouting.id()); return new ShardSegments(indexShard.routingEntry(), indexShard.segments(request.verbose())); } }
/** * Finds the routing source node for peer recovery, return null if its not found. Note, this method expects the shard * routing to *require* peer recovery, use {@link ShardRouting#recoverySource()} to check if its needed or not. */ private static DiscoveryNode findSourceNodeForPeerRecovery(Logger logger, RoutingTable routingTable, DiscoveryNodes nodes, ShardRouting shardRouting) { DiscoveryNode sourceNode = null; if (!shardRouting.primary()) { ShardRouting primary = routingTable.shardRoutingTable(shardRouting.shardId()).primaryShard(); // only recover from started primary, if we can't find one, we will do it next round if (primary.active()) { sourceNode = nodes.get(primary.currentNodeId()); if (sourceNode == null) { logger.trace("can't find replica source node because primary shard {} is assigned to an unknown node.", primary); } } else { logger.trace("can't find replica source node because primary shard {} is not active.", primary); } } else if (shardRouting.relocatingNodeId() != null) { sourceNode = nodes.get(shardRouting.relocatingNodeId()); if (sourceNode == null) { logger.trace("can't find relocation source node for shard {} because it is assigned to an unknown node [{}].", shardRouting.shardId(), shardRouting.relocatingNodeId()); } } else { throw new IllegalStateException("trying to find source node for peer recovery when routing state means no peer recovery: " + shardRouting); } return sourceNode; }
/** * Cancel relocation of a shard. The shards state must be set * to <code>RELOCATING</code>. */ public ShardRouting cancelRelocation() { assert state == ShardRoutingState.RELOCATING : this; assert assignedToNode() : this; assert relocatingNodeId != null : this; return new ShardRouting(shardId, currentNodeId, null, primary, ShardRoutingState.STARTED, recoverySource, null, AllocationId.cancelRelocation(allocationId), UNAVAILABLE_EXPECTED_SHARD_SIZE); }
/** * Method that incorporates the ShardId for the shard into a string that * includes a 'p' or 'r' depending on whether the shard is a primary. */ static String shardIdentifierFromRouting(ShardRouting shardRouting) { return shardRouting.shardId().toString() + "[" + (shardRouting.primary() ? "p" : "r") + "]"; } }
private int numDocsOnPrimary(List<ShardRouting> shards, Map<String, PreSyncedFlushResponse> preSyncResponses) { for (ShardRouting shard : shards) { if (shard.primary()) { final PreSyncedFlushResponse resp = preSyncResponses.get(shard.currentNodeId()); if (resp != null) { return resp.numDocs; } } } return PreSyncedFlushResponse.UNKNOWN_NUM_DOCS; }
throw new IllegalStateException("Can't create shard " + routing.shardId() + ", closed"); final ShardId shardId = routing.shardId(); boolean success = false; Store store = null; Path dataPath = shard.shardPath().getRootStatePath(); Integer curCount = dataPathToShardCount.get(dataPath); if (curCount == null) { routing.getExpectedShardSize() == ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE ? getAvgShardSizeInBytes() : routing.getExpectedShardSize(), dataPathToShardCount); logger.debug("{} creating using a new path [{}]", shardId, path); } else { logger.debug("{} creating using an existing path [{}]", shardId, path); if (shards.containsKey(shardId.id())) { throw new IllegalStateException(shardId + " already exists"); IndexShard shard = getShardOrNull(shardId.getId()); if (shard != null) { warmer.warm(searcher, shard, IndexService.this.indexSettings);
private RecoveryResponse recover(final StartRecoveryRequest request) throws IOException { final IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex()); final IndexShard shard = indexService.getShard(request.shardId().id()); final ShardRouting routingEntry = shard.routingEntry(); if (routingEntry.primary() == false || routingEntry.active() == false) { throw new DelayRecoveryException("source shard [" + routingEntry + "] is not an active primary"); } if (request.isPrimaryRelocation() && (routingEntry.relocating() == false || routingEntry.relocatingNodeId().equals(request.targetNode().getId()) == false)) { logger.debug("delaying recovery of {} as source shard is not marked yet as relocating to {}", request.shardId(), request.targetNode()); throw new DelayRecoveryException("source shard is not marked yet as relocating to [" + request.targetNode() + "]"); } RecoverySourceHandler handler = ongoingRecoveries.addNewRecovery(request, shard); logger.trace("[{}][{}] starting recovery to {}", request.shardId().getIndex().getName(), request.shardId().id(), request.targetNode()); try { return handler.recoverToTarget(); } finally { ongoingRecoveries.remove(shard, handler); } }
private void createShard(DiscoveryNodes nodes, RoutingTable routingTable, ShardRouting shardRouting, ClusterState state) { assert shardRouting.initializing() : "only allow shard creation for initializing shard but was " + shardRouting; DiscoveryNode sourceNode = null; if (shardRouting.recoverySource().getType() == Type.PEER) { sourceNode = findSourceNodeForPeerRecovery(logger, routingTable, nodes, shardRouting); if (sourceNode == null) { logger.trace("ignoring initializing shard {} - no source node can be found.", shardRouting.shardId()); return; } } try { logger.debug("{} creating shard", shardRouting.shardId()); RecoveryState recoveryState = new RecoveryState(shardRouting, nodes.getLocalNode(), sourceNode); indicesService.createShard(shardRouting, recoveryState, recoveryTargetService, new RecoveryListener(shardRouting), repositoriesService, failedShardHandler, globalCheckpointSyncer); } catch (Exception e) { failAndRemoveShard(shardRouting, true, "failed to create shard", e, state); } }
for (RoutingNode routingNode : routingNodes) { for (ShardRouting shard : routingNode) { if (shard.primary()) { continue; if (shard.initializing() == false) { continue; if (shard.relocatingNodeId() != null) { continue; if (shard.unassignedInfo() != null && shard.unassignedInfo().getReason() == UnassignedInfo.Reason.INDEX_CREATED) { continue; logger.trace("{}: fetching new stores for initializing shard", shard); continue; // still fetching ShardRouting primaryShard = allocation.routingNodes().activePrimary(shard.shardId()); assert primaryShard != null : "the replica shard can be allocated on at least one node, so there must be an active primary"; TransportNodesListShardStoreMetaData.StoreFilesMetaData primaryStore = findStore(primaryShard, allocation, shardStores); DiscoveryNode currentNode = allocation.nodes().get(shard.currentNodeId()); DiscoveryNode nodeWithHighestMatch = matchingNodes.getNodeWithHighestMatch(); currentSyncId = null; metaData.getIndexSafe(shard.index()), allocation.changes()));
private void createIndices(final ClusterState state) { RoutingNode localRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId()); if (localRoutingNode == null) { return; if (failedShardsCache.containsKey(shardRouting.shardId()) == false) { final Index index = shardRouting.index(); if (indicesService.indexService(index) == null) { indicesToCreate.computeIfAbsent(index, k -> new ArrayList<>()).add(shardRouting); final IndexMetaData indexMetaData = state.metaData().index(index); logger.debug("[{}] creating index", index); indexService = indicesService.createIndex(indexMetaData, buildInIndexListener); if (indexService.updateMapping(null, indexMetaData) && sendRefreshMapping) { nodeMappingRefreshAction.nodeMappingRefresh(state.nodes().getMasterNode(), new NodeMappingRefreshAction.NodeMappingRefreshRequest(indexMetaData.getIndex().getName(), indexMetaData.getIndexUUID(), state.nodes().getLocalNodeId()) );
final String localNodeId = state.nodes().getLocalNodeId(); assert localNodeId != null; RoutingNode localRoutingNode = state.getRoutingNodes().node(localNodeId); if (localRoutingNode != null) { // null e.g. if we are not a data node for (ShardRouting shardRouting : localRoutingNode) { indicesWithShards.add(shardRouting.index()); final IndexMetaData indexMetaData = state.metaData().index(index); assert indexMetaData != null || event.isNewCluster() : "index " + index + " does not exist in the cluster state, it should either " + final AllocatedIndices.IndexRemovalReason reason = indexMetaData != null && indexMetaData.getState() == IndexMetaData.State.CLOSE ? CLOSED : NO_LONGER_ASSIGNED; logger.debug("{} removing index, [{}]", index, reason); indicesService.removeIndex(index, reason, "removing index (no shards allocated)");
final String localNodeId = state.nodes().getLocalNodeId(); assert localNodeId != null; RoutingNode localRoutingNode = state.getRoutingNodes().node(localNodeId); for (AllocatedIndex<? extends Shard> indexService : indicesService) { for (Shard shard : indexService) { ShardRouting currentRoutingEntry = shard.routingEntry(); ShardId shardId = currentRoutingEntry.shardId(); ShardRouting newShardRouting = localRoutingNode == null ? null : localRoutingNode.getByShardId(shardId); if (newShardRouting == null) { logger.debug("{} removing shard (not allocated)", shardId); indexService.removeShard(shardId.id(), "removing shard (not allocated)"); } else if (newShardRouting.isSameAllocation(currentRoutingEntry) == false) { logger.debug("{} removing shard (stale allocation id, stale {}, new {})", shardId, currentRoutingEntry, newShardRouting); indexService.removeShard(shardId.id(), "removing shard (stale copy)"); } else if (newShardRouting.initializing() && currentRoutingEntry.active()) { logger.debug("{} removing shard (not active, current {}, new {})", shardId, currentRoutingEntry, newShardRouting); indexService.removeShard(shardId.id(), "removing shard (stale copy)"); } else if (newShardRouting.primary() && currentRoutingEntry.primary() == false && newShardRouting.initializing()) { assert currentRoutingEntry.initializing() : currentRoutingEntry; // see above if clause
private void failAndRemoveShard(ShardRouting shardRouting, boolean sendShardFailure, String message, @Nullable Exception failure, ClusterState state) { try { AllocatedIndex<? extends Shard> indexService = indicesService.indexService(shardRouting.shardId().getIndex()); if (indexService != null) { indexService.removeShard(shardRouting.shardId().id(), message); } } catch (ShardNotFoundException e) { // the node got closed on us, ignore it } catch (Exception inner) { inner.addSuppressed(failure); logger.warn(() -> new ParameterizedMessage( "[{}][{}] failed to remove shard after failure ([{}])", shardRouting.getIndexName(), shardRouting.getId(), message), inner); } if (sendShardFailure) { sendFailShard(shardRouting, message, failure, state); } }
private void sendFailShard(ShardRouting shardRouting, String message, @Nullable Exception failure, ClusterState state) { try { logger.warn(() -> new ParameterizedMessage( "[{}] marking and sending shard failed due to [{}]", shardRouting.shardId(), message), failure); failedShardsCache.put(shardRouting.shardId(), shardRouting); shardStateAction.localShardFailed(shardRouting, message, failure, SHARD_STATE_ACTION_LISTENER, state); } catch (Exception inner) { if (failure != null) inner.addSuppressed(failure); logger.warn(() -> new ParameterizedMessage( "[{}][{}] failed to mark shard as failed (because of [{}])", shardRouting.getIndexName(), shardRouting.getId(), message), inner); } }
Runnable globalCheckpointSyncer, CircuitBreakerService circuitBreakerService) throws IOException { super(shardRouting.shardId(), indexSettings); assert shardRouting.initializing(); this.shardRouting = shardRouting; final Settings settings = indexSettings.getSettings(); this.circuitBreakerService = circuitBreakerService; logger.debug("state: [CREATED]"); + "which has no effect and will not be accepted in future"); this.translogConfig = new TranslogConfig(shardId, shardPath().resolveTranslog(), indexSettings, bigArrays); final String aId = shardRouting.allocationId().getId(); this.globalCheckpointListeners = new GlobalCheckpointListeners(shardId, threadPool.executor(ThreadPool.Names.LISTENER), threadPool.scheduler(), logger); pendingPrimaryTerm = indexSettings.getIndexMetaData().primaryTerm(shardId.id()); operationPrimaryTerm = pendingPrimaryTerm; refreshListeners = buildRefreshListeners(); persistMetadata(path, indexSettings, shardRouting, null, logger);
private void updateShard(DiscoveryNodes nodes, ShardRouting shardRouting, Shard shard, RoutingTable routingTable, ClusterState clusterState) { final ShardRouting currentRoutingEntry = shard.routingEntry(); assert currentRoutingEntry.isSameAllocation(shardRouting) : "local shard has a different allocation id but wasn't cleaning by removeShards. " + "cluster state: " + shardRouting + " local: " + currentRoutingEntry; final IndexMetaData indexMetaData = clusterState.metaData().index(shard.shardId().getIndex()); final long primaryTerm = indexMetaData.primaryTerm(shard.shardId().id()); final Set<String> inSyncIds = indexMetaData.inSyncAllocationIds(shard.shardId().id()); final IndexShardRoutingTable indexShardRoutingTable = routingTable.shardRoutingTable(shardRouting.shardId()); final Set<String> pre60AllocationIds = indexShardRoutingTable.assignedShards() .stream() .flatMap(shr -> { if (shr.relocating()) { return Stream.of(shr, shr.getTargetRelocatingShard()); } else { return Stream.of(shr); .filter(shr -> nodes.get(shr.currentNodeId()).getVersion().before(Version.V_6_0_0_alpha1)) .map(ShardRouting::allocationId) .map(AllocationId::getId) .collect(Collectors.toSet()); shard.updateShardState(shardRouting, primaryTerm, primaryReplicaSyncer::resync, clusterState.version(), inSyncIds, indexShardRoutingTable, pre60AllocationIds); } catch (Exception e) { if (shardRouting.initializing() && (state == IndexShardState.STARTED || state == IndexShardState.POST_RECOVERY)) { if (logger.isTraceEnabled()) {
final RecoveryState.Translog translogState = indexShard.recoveryState().getTranslog(); if (restoreSource == null) { throw new IndexShardRestoreFailedException(shardId, "empty restore source"); if (logger.isTraceEnabled()) { logger.trace("[{}] restoring shard [{}]", restoreSource.snapshot(), shardId); indexShard.prepareForIndexRecovery(); ShardId snapshotShardId = shardId; final String indexName = restoreSource.index(); if (!shardId.getIndexName().equals(indexName)) { snapshotShardId = new ShardId(indexName, IndexMetaData.INDEX_UUID_NA_VALUE, shardId.id()); restoreSource.version(), indexId, snapshotShardId, indexShard.recoveryState()); final Store store = indexShard.store(); store.bootstrapNewHistory(); indexShard.shardPath().resolveTranslog(), maxSeqNo, shardId, indexShard.getPendingPrimaryTerm()); store.associateIndexWithNewTranslog(translogUUID); assert indexShard.shardRouting.primary() : "only primary shards can recover from store"; indexShard.openEngineAndRecoverFromTranslog(); indexShard.getEngine().fillSeqNoGaps(indexShard.getPendingPrimaryTerm());
final ConcurrentMap<String, PreSyncedFlushResponse> presyncResponses = ConcurrentCollections.newConcurrentMap(); for (final ShardRouting shard : shards) { logger.trace("{} sending pre-synced flush request to {}", shardId, shard); final DiscoveryNode node = state.nodes().get(shard.currentNodeId()); if (node == null) { logger.trace("{} shard routing {} refers to an unknown node. skipping.", shardId, shard); if (countDown.countDown()) { listener.onResponse(presyncResponses); transportService.sendRequest(node, PRE_SYNCED_FLUSH_ACTION_NAME, new PreShardSyncedFlushRequest(shard.shardId()), new TransportResponseHandler<PreSyncedFlushResponse>() { @Override
private void deleteShardIfExistElseWhere(ClusterState state, IndexShardRoutingTable indexShardRoutingTable) { List<Tuple<DiscoveryNode, ShardActiveRequest>> requests = new ArrayList<>(indexShardRoutingTable.size()); String indexUUID = indexShardRoutingTable.shardId().getIndex().getUUID(); ClusterName clusterName = state.getClusterName(); for (ShardRouting shardRouting : indexShardRoutingTable) { assert shardRouting.started() : "expected started shard but was " + shardRouting; DiscoveryNode currentNode = state.nodes().get(shardRouting.currentNodeId()); requests.add(new Tuple<>(currentNode, new ShardActiveRequest(clusterName, indexUUID, shardRouting.shardId(), deleteShardTimeout))); } ShardActiveResponseHandler responseHandler = new ShardActiveResponseHandler(indexShardRoutingTable.shardId(), state.getVersion(), requests.size()); for (Tuple<DiscoveryNode, ShardActiveRequest> request : requests) { logger.trace("{} sending shard active check to {}", request.v2().shardId, request.v1()); transportService.sendRequest(request.v1(), ACTION_SHARD_EXISTS, request.v2(), responseHandler); } }