/** * Returns true if no primaries are active or initializing for this shard */ private boolean noPrimariesActive() { if (!primaryAsList.isEmpty() && !primaryAsList.get(0).active() && !primaryAsList.get(0).initializing()) { return true; } return false; }
@Override public void shardInitialized(ShardRouting unassignedShard, ShardRouting initializedShard) { assert unassignedShard.unassigned() : "expected unassigned shard " + unassignedShard; assert initializedShard.initializing() : "expected initializing shard " + initializedShard; setChanged(); }
@Override public void shardStarted(ShardRouting initializingShard, ShardRouting startedShard) { assert initializingShard.initializing() : "expected initializing shard " + initializingShard; assert startedShard.started() : "expected started shard " + startedShard; setChanged(); }
public boolean recoverFromStore() { // we are the first primary, recover from the gateway // if its post api allocation, the index should exists assert shardRouting.primary() : "recover from store only makes sense if the shard is a primary shard"; assert shardRouting.initializing() : "can only start recovery on initializing shard"; StoreRecovery storeRecovery = new StoreRecovery(shardId, logger); return storeRecovery.recoverFromStore(this); }
@Override public void initializedReplicaReinitialized(ShardRouting oldReplica, ShardRouting reinitializedReplica) { assert oldReplica.initializing() && oldReplica.primary() == false : "expected initializing replica shard " + oldReplica; assert reinitializedReplica.initializing() && reinitializedReplica.primary() == false : "expected reinitialized replica shard " + reinitializedReplica; assert oldReplica.allocationId().getId().equals(reinitializedReplica.allocationId().getId()) == false : "expected allocation id to change for reinitialized replica shard (old: " + oldReplica + " new: " + reinitializedReplica + ")"; setChanged(); }
public ShardIterator replicaActiveInitializingShardIt() { // If the primaries are unassigned, return an empty list (there aren't // any replicas to query anyway) if (noPrimariesActive()) { return new PlainShardIterator(shardId, NO_SHARDS); } LinkedList<ShardRouting> ordered = new LinkedList<>(); for (ShardRouting replica : shuffler.shuffle(replicas)) { if (replica.active()) { ordered.addFirst(replica); } else if (replica.initializing()) { ordered.addLast(replica); } } return new PlainShardIterator(shardId, ordered); }
@Override public void startedPrimaryReinitialized(ShardRouting startedPrimaryShard, ShardRouting initializedShard) { assert startedPrimaryShard.primary() && startedPrimaryShard.started() : "expected started primary shard " + startedPrimaryShard; assert initializedShard.primary() && initializedShard.initializing(): "expected initializing primary shard " + initializedShard; setChanged(); }
private ShardRouting reinitReplica(ShardRouting shard) { assert shard.primary() == false : "shard must be a replica: " + shard; assert shard.initializing() : "can only reinitialize an initializing replica: " + shard; assert shard.isRelocationTarget() == false : "replication target cannot be reinitialized: " + shard; ShardRouting reinitializedShard = shard.reinitializeReplicaShard(); updateAssigned(shard, reinitializedShard); return reinitializedShard; }
@Override public void shardFailed(ShardRouting failedShard, UnassignedInfo unassignedInfo) { if (failedShard.primary() && failedShard.initializing()) { RecoverySource recoverySource = failedShard.recoverySource(); if (recoverySource.getType() == RecoverySource.Type.SNAPSHOT) { // mark restore entry for this shard as failed when it's due to a file corruption. There is no need wait on retries // to restore this shard on another node if the snapshot files are corrupt. In case where a node just left or crashed, // however, we only want to acknowledge the restore operation once it has been successfully restored on another node. if (unassignedInfo.getFailure() != null && Lucene.isCorruptionException(unassignedInfo.getFailure().getCause())) { changes(recoverySource).shards.put( failedShard.shardId(), new ShardRestoreStatus(failedShard.currentNodeId(), RestoreInProgress.State.FAILURE, unassignedInfo.getFailure().getCause().getMessage())); } } } }
/** * Mark a shard as started and adjusts internal statistics. * * @return the started shard */ private ShardRouting started(ShardRouting shard) { assert shard.initializing() : "expected an initializing shard " + shard; if (shard.relocatingNodeId() == null) { // if this is not a target shard for relocation, we need to update statistics inactiveShardCount--; if (shard.primary()) { inactivePrimaryCount--; } } removeRecovery(shard); ShardRouting startedShard = shard.moveToStarted(); updateAssigned(shard, startedShard); return startedShard; }
/** * Cancels the give shard from the Routing nodes internal statistics and cancels * the relocation if the shard is relocating. */ private void remove(ShardRouting shard) { assert shard.unassigned() == false : "only assigned shards can be removed here (" + shard + ")"; node(shard.currentNodeId()).remove(shard); if (shard.initializing() && shard.relocatingNodeId() == null) { inactiveShardCount--; assert inactiveShardCount >= 0; if (shard.primary()) { inactivePrimaryCount--; } } else if (shard.relocating()) { shard = cancelRelocation(shard); } assignedShardsRemove(shard); if (shard.initializing()) { removeRecovery(shard); } }
/** * Notifies master about shards that don't exist but are supposed to be active on this node. * * @param state new cluster state */ private void failMissingShards(final ClusterState state) { RoutingNode localRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId()); if (localRoutingNode == null) { return; } for (final ShardRouting shardRouting : localRoutingNode) { ShardId shardId = shardRouting.shardId(); if (shardRouting.initializing() == false && failedShardsCache.containsKey(shardId) == false && indicesService.getShardOrNull(shardId) == null) { // the master thinks we are active, but we don't have this shard at all, mark it as failed sendFailShard(shardRouting, "master marked shard as active, but shard has not been created, mark shard as failed", null, state); } } }
/** * Returns the size of all shards that are currently being relocated to * the node, but may not be finished transferring yet. * * If subtractShardsMovingAway is true then the size of shards moving away is subtracted from the total size of all shards */ static long sizeOfRelocatingShards(RoutingNode node, RoutingAllocation allocation, boolean subtractShardsMovingAway, String dataPath) { ClusterInfo clusterInfo = allocation.clusterInfo(); long totalSize = 0; for (ShardRouting routing : node.shardsWithState(ShardRoutingState.RELOCATING, ShardRoutingState.INITIALIZING)) { String actualPath = clusterInfo.getDataPath(routing); if (dataPath.equals(actualPath)) { if (routing.initializing() && routing.relocatingNodeId() != null) { totalSize += getExpectedShardSize(routing, allocation, 0); } else if (subtractShardsMovingAway && routing.relocating()) { totalSize -= getExpectedShardSize(routing, allocation, 0); } } } return totalSize; }
private void createShard(DiscoveryNodes nodes, RoutingTable routingTable, ShardRouting shardRouting, ClusterState state) { assert shardRouting.initializing() : "only allow shard creation for initializing shard but was " + shardRouting; DiscoveryNode sourceNode = null; if (shardRouting.recoverySource().getType() == Type.PEER) { sourceNode = findSourceNodeForPeerRecovery(logger, routingTable, nodes, shardRouting); if (sourceNode == null) { logger.trace("ignoring initializing shard {} - no source node can be found.", shardRouting.shardId()); return; } } try { logger.debug("{} creating shard", shardRouting.shardId()); RecoveryState recoveryState = new RecoveryState(shardRouting, nodes.getLocalNode(), sourceNode); indicesService.createShard(shardRouting, recoveryState, recoveryTargetService, new RecoveryListener(shardRouting), repositoriesService, failedShardHandler, globalCheckpointSyncer); } catch (Exception e) { failAndRemoveShard(shardRouting, true, "failed to create shard", e, state); } }
public RecoveryState(ShardRouting shardRouting, DiscoveryNode targetNode, @Nullable DiscoveryNode sourceNode) { assert shardRouting.initializing() : "only allow initializing shard routing to be recovered: " + shardRouting; RecoverySource recoverySource = shardRouting.recoverySource(); assert (recoverySource.getType() == RecoverySource.Type.PEER) == (sourceNode != null) : "peer recovery requires source node, recovery type: " + recoverySource.getType() + " source node: " + sourceNode; this.shardId = shardRouting.shardId(); this.primary = shardRouting.primary(); this.recoverySource = recoverySource; this.sourceNode = sourceNode; this.targetNode = targetNode; stage = Stage.INIT; timer.start(); }
private void updateRecoveryCounts(final ShardRouting routing, final boolean increment, @Nullable final ShardRouting primary) { final int howMany = increment ? 1 : -1; assert routing.initializing() : "routing must be initializing: " + routing; // TODO: check primary == null || primary.active() after all tests properly add ReplicaAfterPrimaryActiveAllocationDecider assert primary == null || primary.assignedToNode() : "shard is initializing but its primary is not assigned to a node"; Recoveries.getOrAdd(recoveriesPerNode, routing.currentNodeId()).addIncoming(howMany); if (routing.recoverySource().getType() == RecoverySource.Type.PEER) { // add/remove corresponding outgoing recovery on node with primary shard if (primary == null) { throw new IllegalStateException("shard is peer recovering but primary is unassigned"); } Recoveries.getOrAdd(recoveriesPerNode, primary.currentNodeId()).addOutgoing(howMany); if (increment == false && routing.primary() && routing.relocatingNodeId() != null) { // primary is done relocating, move non-primary recoveries from old primary to new primary int numRecoveringReplicas = 0; for (ShardRouting assigned : assignedShards(routing.shardId())) { if (assigned.primary() == false && assigned.initializing() && assigned.recoverySource().getType() == RecoverySource.Type.PEER) { numRecoveringReplicas++; } } recoveriesPerNode.get(routing.relocatingNodeId()).addOutgoing(-numRecoveringReplicas); recoveriesPerNode.get(routing.currentNodeId()).addOutgoing(numRecoveringReplicas); } } }
if (shardRouting.unassigned()) { initializingShard = shardRouting.initialize(currentNodeId, null, ShardRouting.UNAVAILABLE_EXPECTED_SHARD_SIZE); } else if (shardRouting.initializing()) { UnassignedInfo unassignedInfo = shardRouting.unassignedInfo(); if (unassignedInfo == null) { .getTargetRelocatingShard(); assert initializingShard.initializing(); return initializingShard;
private void applyStartedShards(RoutingAllocation routingAllocation, List<ShardRouting> startedShardEntries) { assert startedShardEntries.isEmpty() == false : "non-empty list of started shard entries expected"; RoutingNodes routingNodes = routingAllocation.routingNodes(); for (ShardRouting startedShard : startedShardEntries) { assert startedShard.initializing() : "only initializing shards can be started"; assert routingAllocation.metaData().index(startedShard.shardId().getIndex()) != null : "shard started for unknown index (shard entry: " + startedShard + ")"; assert startedShard == routingNodes.getByAllocationId(startedShard.shardId(), startedShard.allocationId().getId()) : "shard routing to start does not exist in routing table, expected: " + startedShard + " but was: " + routingNodes.getByAllocationId(startedShard.shardId(), startedShard.allocationId().getId()); routingNodes.startShard(logger, startedShard, routingAllocation.changes()); } }
private void createOrUpdateShards(final ClusterState state) { RoutingNode localRoutingNode = state.getRoutingNodes().node(state.nodes().getLocalNodeId()); if (localRoutingNode == null) { return; } DiscoveryNodes nodes = state.nodes(); RoutingTable routingTable = state.routingTable(); for (final ShardRouting shardRouting : localRoutingNode) { ShardId shardId = shardRouting.shardId(); if (failedShardsCache.containsKey(shardId) == false) { AllocatedIndex<? extends Shard> indexService = indicesService.indexService(shardId.getIndex()); assert indexService != null : "index " + shardId.getIndex() + " should have been created by createIndices"; Shard shard = indexService.getShardOrNull(shardId.id()); if (shard == null) { assert shardRouting.initializing() : shardRouting + " should have been removed by failMissingShards"; createShard(nodes, routingTable, shardRouting, state); } else { updateShard(nodes, shardRouting, shard, routingTable, state); } } } }
public static ClusterAllocationExplanation explainShard(ShardRouting shardRouting, RoutingAllocation allocation, ClusterInfo clusterInfo, boolean includeYesDecisions, GatewayAllocator gatewayAllocator, ShardsAllocator shardAllocator) { allocation.setDebugMode(includeYesDecisions ? DebugMode.ON : DebugMode.EXCLUDE_YES_DECISIONS); ShardAllocationDecision shardDecision; if (shardRouting.initializing() || shardRouting.relocating()) { shardDecision = ShardAllocationDecision.NOT_TAKEN; } else { AllocateUnassignedDecision allocateDecision = shardRouting.unassigned() ? gatewayAllocator.decideUnassignedShardAllocation(shardRouting, allocation) : AllocateUnassignedDecision.NOT_TAKEN; if (allocateDecision.isDecisionTaken() == false) { shardDecision = shardAllocator.decideShardAllocation(shardRouting, allocation); } else { shardDecision = new ShardAllocationDecision(allocateDecision, MoveDecision.NOT_TAKEN); } } return new ClusterAllocationExplanation(shardRouting, shardRouting.currentNodeId() != null ? allocation.nodes().get(shardRouting.currentNodeId()) : null, shardRouting.relocatingNodeId() != null ? allocation.nodes().get(shardRouting.relocatingNodeId()) : null, clusterInfo, shardDecision); }