static boolean distinctNodes(List<ShardRouting> shards) { Set<String> nodes = new HashSet<>(); for (ShardRouting shard : shards) { if (shard.assignedToNode()) { if (nodes.add(shard.currentNodeId()) == false) { return false; } if (shard.relocating()) { if (nodes.add(shard.relocatingNodeId()) == false) { return false; } } } } return true; }
/** * Finds the routing source node for peer recovery, return null if its not found. Note, this method expects the shard * routing to *require* peer recovery, use {@link ShardRouting#recoverySource()} to check if its needed or not. */ private static DiscoveryNode findSourceNodeForPeerRecovery(Logger logger, RoutingTable routingTable, DiscoveryNodes nodes, ShardRouting shardRouting) { DiscoveryNode sourceNode = null; if (!shardRouting.primary()) { ShardRouting primary = routingTable.shardRoutingTable(shardRouting.shardId()).primaryShard(); // only recover from started primary, if we can't find one, we will do it next round if (primary.active()) { sourceNode = nodes.get(primary.currentNodeId()); if (sourceNode == null) { logger.trace("can't find replica source node because primary shard {} is assigned to an unknown node.", primary); } } else { logger.trace("can't find replica source node because primary shard {} is not active.", primary); } } else if (shardRouting.relocatingNodeId() != null) { sourceNode = nodes.get(shardRouting.relocatingNodeId()); if (sourceNode == null) { logger.trace("can't find relocation source node for shard {} because it is assigned to an unknown node [{}].", shardRouting.shardId(), shardRouting.relocatingNodeId()); } } else { throw new IllegalStateException("trying to find source node for peer recovery when routing state means no peer recovery: " + shardRouting); } return sourceNode; }
/** * Mark a shard as started and adjusts internal statistics. * * @return the started shard */ private ShardRouting started(ShardRouting shard) { assert shard.initializing() : "expected an initializing shard " + shard; if (shard.relocatingNodeId() == null) { // if this is not a target shard for relocation, we need to update statistics inactiveShardCount--; if (shard.primary()) { inactivePrimaryCount--; } } removeRecovery(shard); ShardRouting startedShard = shard.moveToStarted(); updateAssigned(shard, startedShard); return startedShard; }
@Nullable private ShardRouting findAssignedPrimaryIfPeerRecovery(ShardRouting routing) { ShardRouting primary = null; if (routing.recoverySource() != null && routing.recoverySource().getType() == RecoverySource.Type.PEER) { List<ShardRouting> shardRoutings = assignedShards.get(routing.shardId()); if (shardRoutings != null) { for (ShardRouting shardRouting : shardRoutings) { if (shardRouting.primary()) { if (shardRouting.active()) { return shardRouting; } else if (primary == null) { primary = shardRouting; } else if (primary.relocatingNodeId() != null) { primary = shardRouting; } } } } } return primary; }
/** * Returns the size of all shards that are currently being relocated to * the node, but may not be finished transferring yet. * * If subtractShardsMovingAway is true then the size of shards moving away is subtracted from the total size of all shards */ static long sizeOfRelocatingShards(RoutingNode node, RoutingAllocation allocation, boolean subtractShardsMovingAway, String dataPath) { ClusterInfo clusterInfo = allocation.clusterInfo(); long totalSize = 0; for (ShardRouting routing : node.shardsWithState(ShardRoutingState.RELOCATING, ShardRoutingState.INITIALIZING)) { String actualPath = clusterInfo.getDataPath(routing); if (dataPath.equals(actualPath)) { if (routing.initializing() && routing.relocatingNodeId() != null) { totalSize += getExpectedShardSize(routing, allocation, 0); } else if (subtractShardsMovingAway && routing.relocating()) { totalSize -= getExpectedShardSize(routing, allocation, 0); } } } return totalSize; }
@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(Fields.ROUTING) .field(Fields.STATE, shardRouting.state()) .field(Fields.PRIMARY, shardRouting.primary()) .field(Fields.NODE, shardRouting.currentNodeId()) .field(Fields.RELOCATING_NODE, shardRouting.relocatingNodeId()) .endObject(); commonStats.toXContent(builder, params); if (commitStats != null) { commitStats.toXContent(builder, params); } if (seqNoStats != null) { seqNoStats.toXContent(builder, params); } builder.startObject(Fields.SHARD_PATH); builder.field(Fields.STATE_PATH, statePath); builder.field(Fields.DATA_PATH, dataPath); builder.field(Fields.IS_CUSTOM_DATA_PATH, isCustomDataPath); builder.endObject(); return builder; }
if (shard.relocating()) { relocatingShards++; entries = nodesToShards.computeIfAbsent(shard.relocatingNodeId(), k -> new LinkedHashMap<>()); // LinkedHashMap to preserve order
for (ShardRouting shardRoutingEntry : routingNode) { if (shardRoutingEntry.initializing() && shardRoutingEntry.relocatingNodeId() != null) continue;
private void updateRecoveryCounts(final ShardRouting routing, final boolean increment, @Nullable final ShardRouting primary) { final int howMany = increment ? 1 : -1; assert routing.initializing() : "routing must be initializing: " + routing; // TODO: check primary == null || primary.active() after all tests properly add ReplicaAfterPrimaryActiveAllocationDecider assert primary == null || primary.assignedToNode() : "shard is initializing but its primary is not assigned to a node"; Recoveries.getOrAdd(recoveriesPerNode, routing.currentNodeId()).addIncoming(howMany); if (routing.recoverySource().getType() == RecoverySource.Type.PEER) { // add/remove corresponding outgoing recovery on node with primary shard if (primary == null) { throw new IllegalStateException("shard is peer recovering but primary is unassigned"); } Recoveries.getOrAdd(recoveriesPerNode, primary.currentNodeId()).addOutgoing(howMany); if (increment == false && routing.primary() && routing.relocatingNodeId() != null) { // primary is done relocating, move non-primary recoveries from old primary to new primary int numRecoveringReplicas = 0; for (ShardRouting assigned : assignedShards(routing.shardId())) { if (assigned.primary() == false && assigned.initializing() && assigned.recoverySource().getType() == RecoverySource.Type.PEER) { numRecoveringReplicas++; } } recoveriesPerNode.get(routing.relocatingNodeId()).addOutgoing(-numRecoveringReplicas); recoveriesPerNode.get(routing.currentNodeId()).addOutgoing(numRecoveringReplicas); } } }
builder.field(Fields.PRIMARY, shardUpgradeStatus.getShardRouting().primary()); builder.field(Fields.NODE, shardUpgradeStatus.getShardRouting().currentNodeId()); if (shardUpgradeStatus.getShardRouting().relocatingNodeId() != null) { builder.field(Fields.RELOCATING_NODE, shardUpgradeStatus.getShardRouting().relocatingNodeId());
public static ClusterAllocationExplanation explainShard(ShardRouting shardRouting, RoutingAllocation allocation, ClusterInfo clusterInfo, boolean includeYesDecisions, GatewayAllocator gatewayAllocator, ShardsAllocator shardAllocator) { allocation.setDebugMode(includeYesDecisions ? DebugMode.ON : DebugMode.EXCLUDE_YES_DECISIONS); ShardAllocationDecision shardDecision; if (shardRouting.initializing() || shardRouting.relocating()) { shardDecision = ShardAllocationDecision.NOT_TAKEN; } else { AllocateUnassignedDecision allocateDecision = shardRouting.unassigned() ? gatewayAllocator.decideUnassignedShardAllocation(shardRouting, allocation) : AllocateUnassignedDecision.NOT_TAKEN; if (allocateDecision.isDecisionTaken() == false) { shardDecision = shardAllocator.decideShardAllocation(shardRouting, allocation); } else { shardDecision = new ShardAllocationDecision(allocateDecision, MoveDecision.NOT_TAKEN); } } return new ClusterAllocationExplanation(shardRouting, shardRouting.currentNodeId() != null ? allocation.nodes().get(shardRouting.currentNodeId()) : null, shardRouting.relocatingNodeId() != null ? allocation.nodes().get(shardRouting.relocatingNodeId()) : null, clusterInfo, shardDecision); }
builder.field(Fields.PRIMARY, shardSegments.getShardRouting().primary()); builder.field(Fields.NODE, shardSegments.getShardRouting().currentNodeId()); if (shardSegments.getShardRouting().relocatingNodeId() != null) { builder.field(Fields.RELOCATING_NODE, shardSegments.getShardRouting().relocatingNodeId());
@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject() .field("state", state()) .field("primary", primary()) .field("node", currentNodeId()) .field("relocating_node", relocatingNodeId()) .field("shard", id()) .field("index", getIndexName()); if (expectedShardSize != UNAVAILABLE_EXPECTED_SHARD_SIZE) { builder.field("expected_shard_size_in_bytes", expectedShardSize); } if (recoverySource != null) { builder.field("recovery_source", recoverySource); } if (allocationId != null) { builder.field("allocation_id"); allocationId.toXContent(builder, params); } if (unassignedInfo != null) { unassignedInfo.toXContent(builder, params); } return builder.endObject(); }
continue; if (shard.relocatingNodeId() != null) { continue;
/** * Cancels the give shard from the Routing nodes internal statistics and cancels * the relocation if the shard is relocating. */ private void remove(ShardRouting shard) { assert shard.unassigned() == false : "only assigned shards can be removed here (" + shard + ")"; node(shard.currentNodeId()).remove(shard); if (shard.initializing() && shard.relocatingNodeId() == null) { inactiveShardCount--; assert inactiveShardCount >= 0; if (shard.primary()) { inactivePrimaryCount--; } } else if (shard.relocating()) { shard = cancelRelocation(shard); } assignedShardsRemove(shard); if (shard.initializing()) { removeRecovery(shard); } }
routingChangesObserver.shardStarted(initializingShard, startedShard); if (initializingShard.relocatingNodeId() != null) { RoutingNode relocationSourceNode = node(initializingShard.relocatingNodeId()); ShardRouting relocationSourceShard = relocationSourceNode.getByShardId(initializingShard.shardId()); assert relocationSourceShard.isRelocationSourceOf(initializingShard); routingChangesObserver.shardFailed(routing, new UnassignedInfo(UnassignedInfo.Reason.REINITIALIZED, "primary changed")); relocateShard(startedReplica, sourceShard.relocatingNodeId(), sourceShard.getExpectedShardSize(), routingChangesObserver); } else {
private RecoveryResponse recover(final StartRecoveryRequest request) throws IOException { final IndexService indexService = indicesService.indexServiceSafe(request.shardId().getIndex()); final IndexShard shard = indexService.getShard(request.shardId().id()); final ShardRouting routingEntry = shard.routingEntry(); if (routingEntry.primary() == false || routingEntry.active() == false) { throw new DelayRecoveryException("source shard [" + routingEntry + "] is not an active primary"); } if (request.isPrimaryRelocation() && (routingEntry.relocating() == false || routingEntry.relocatingNodeId().equals(request.targetNode().getId()) == false)) { logger.debug("delaying recovery of {} as source shard is not marked yet as relocating to {}", request.shardId(), request.targetNode()); throw new DelayRecoveryException("source shard is not marked yet as relocating to [" + request.targetNode() + "]"); } RecoverySourceHandler handler = ongoingRecoveries.addNewRecovery(request, shard); logger.trace("[{}][{}] starting recovery to {}", request.shardId().getIndex().getName(), request.shardId().id(), request.targetNode()); try { return handler.recoverToTarget(); } finally { ongoingRecoveries.remove(shard, handler); } }
if ((shardRouting.initializing() && shardRouting.relocatingNodeId() != null) == false) {
String nodeId = shardRouting.relocating() ? shardRouting.relocatingNodeId() : shardRouting.currentNodeId(); if (!node.nodeId().equals(nodeId)) {
if (shard.initializing() && shard.primary() && shard.relocatingNodeId() == null) { primariesInRecovery++;