Map<String, Replica> shards = entry.getValue().getReplicasMap(); for (Map.Entry<String, Replica> shard : shards.entrySet()) { String state = shard.getValue().getStr(ZkStateReader.STATE_PROP); if ((state.equals(Replica.State.RECOVERING) || state.equals(Replica.State.DOWN)) && clusterState.liveNodesContain(shard.getValue().getStr( ZkStateReader.NODE_NAME_PROP))) { sawLiveRecovering = true;
if (verbose) System.out.println("replica:" + shard.getValue().getName() + " rstate:" + shard.getValue().getStr(ZkStateReader.STATE_PROP) + " live:" + clusterState.liveNodesContain(shard.getValue().getNodeName())); final Replica.State state = shard.getValue().getState(); if ((state == Replica.State.RECOVERING || state == Replica.State.DOWN || state == Replica.State.RECOVERY_FAILED) && clusterState.liveNodesContain(shard.getValue().getStr(ZkStateReader.NODE_NAME_PROP))) { sawLiveRecovering = true;
for (String sliceName : slices.keySet()) { for (Replica replica : slices.get(sliceName).getReplicas()) { if (nodesAllowedToRunShards != null && !nodesAllowedToRunShards.contains(replica.getStr(ZkStateReader.NODE_NAME_PROP))) { return "Shard " + replica.getName() + " created on node " + replica.getNodeName() + " not allowed to run shards for the created collection " + collectionName;
protected List<String> getShardUrls() throws IOException { try { ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader(); Slice[] slices = CloudSolrStream.getSlices(this.collection, zkStateReader, false); ClusterState clusterState = zkStateReader.getClusterState(); Set<String> liveNodes = clusterState.getLiveNodes(); List<String> baseUrls = new ArrayList<>(); for(Slice slice : slices) { Collection<Replica> replicas = slice.getReplicas(); List<Replica> shuffler = new ArrayList<>(); for(Replica replica : replicas) { if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) { shuffler.add(replica); } } Collections.shuffle(shuffler, new Random()); Replica rep = shuffler.get(0); ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep); String url = zkProps.getCoreUrl(); baseUrls.add(url); } return baseUrls; } catch (Exception e) { throw new IOException(e); } }
private static void logCollectionStateSummary(String collectionName, ZkStateReader zkStateReader) { Pattern portPattern = Pattern.compile(".*:([0-9]*).*"); DocCollection docCollection = zkStateReader.getClusterState().getCollection(collectionName); if (docCollection == null) { monkeyLog("Could not find collection {}", collectionName); } StringBuilder builder = new StringBuilder(); builder.append("Collection status: {"); for (Slice slice:docCollection.getSlices()) { builder.append(slice.getName() + ": {"); for (Replica replica:slice.getReplicas()) { log.info(replica.toString()); java.util.regex.Matcher m = portPattern.matcher(replica.getBaseUrl()); m.find(); String jettyPort = m.group(1); builder.append(String.format(Locale.ROOT, "%s(%s): {state: %s, type: %s, leader: %s, Live: %s}, ", replica.getName(), jettyPort, replica.getState(), replica.getType(), (replica.get("leader")!= null), zkStateReader.getClusterState().liveNodesContain(replica.getNodeName()))); } if (slice.getReplicas().size() > 0) { builder.setLength(builder.length() - 2); } builder.append("}, "); } if (docCollection.getSlices().size() > 0) { builder.setLength(builder.length() - 2); } builder.append("}"); monkeyLog(builder.toString()); }
public ReplicaInfo(String coll, String shard, Replica r, Map<String, Object> vals) { this.name = r.getName(); this.core = r.getCoreName(); this.collection = coll; this.shard = shard; this.type = r.getType(); boolean maybeLeader = r.getBool(LEADER_PROP, false); if (vals != null) { this.variables.putAll(vals); maybeLeader = "true".equals(String.valueOf(vals.getOrDefault(LEADER_PROP, maybeLeader))); } this.isLeader = maybeLeader; this.node = r.getNodeName(); }
private void getPersistedCheckpoints() throws IOException { ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader(); Slice[] slices = CloudSolrStream.getSlices(checkpointCollection, zkStateReader, false); ClusterState clusterState = zkStateReader.getClusterState(); Set<String> liveNodes = clusterState.getLiveNodes(); OUTER: for(Slice slice : slices) { Collection<Replica> replicas = slice.getReplicas(); for(Replica replica : replicas) { if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())){ HttpSolrClient httpClient = streamContext.getSolrClientCache().getHttpSolrClient(replica.getCoreUrl()); try { SolrDocument doc = httpClient.getById(id); if(doc != null) { List<String> checkpoints = (List<String>)doc.getFieldValue("checkpoint_ss"); for (String checkpoint : checkpoints) { String[] pair = checkpoint.split("~"); this.checkpoints.put(pair[0], Long.parseLong(pair[1])); } } } catch (Exception e) { throw new IOException(e); } break OUTER; } } } }
private void addNodeNameReplica(Replica replica) { List<Replica> replicas = nodeNameReplicas.get(replica.getNodeName()); if (replicas == null) { replicas = new ArrayList<>(); nodeNameReplicas.put(replica.getNodeName(), replicas); } replicas.add(replica); if (replica.getStr(Slice.LEADER) != null) { List<Replica> leaderReplicas = nodeNameLeaderReplicas.get(replica.getNodeName()); if (leaderReplicas == null) { leaderReplicas = new ArrayList<>(); nodeNameLeaderReplicas.put(replica.getNodeName(), leaderReplicas); } leaderReplicas.add(replica); } }
assertNotNull(leader); log.info("Found "+replicas.size()+" replicas and leader on "+ leader.getNodeName()+" for "+shardId+" in "+testCollectionName); if (!zkShardTerms.canBecomeLeader(replica.getName()) || replica.getState() != Replica.State.ACTIVE) { log.info("Replica {} is currently {}", replica.getName(), replica.getState()); allReplicasUp = false; if (!leader.equals(replica)) notLeaders.put(replica.getName(), replica);
if (Replica.State.ACTIVE.equals(replica.getState())) { sliceHealthy = true; break; } else { LOGGER.info("Replica {} on node {} is {}", replica.getName(), replica.getNodeName(), replica.getState().name());
static ReplicaInfo create(Replica replica) { return new AutoValue_SolrIO_ReplicaInfo( replica.getStr(ZkStateReader.CORE_NAME_PROP), replica.getCoreUrl(), replica.getStr(ZkStateReader.BASE_URL_PROP)); } }
if (directUpdatesToLeadersOnly && leader == null) { for (Replica replica : slice.getReplicas( replica -> replica.isActive(getClusterStateProvider().getLiveNodes()) && replica.getType() == Replica.Type.NRT)) { leader = replica; break; if (!directUpdatesToLeadersOnly) { for (Replica replica : slice.getReplicas()) { if (!replica.getNodeName().equals(leader.getNodeName()) && !replica.getName().equals(leader.getName())) { ZkCoreNodeProps zkProps1 = new ZkCoreNodeProps(replica); String url1 = zkProps1.getCoreUrl();
protected int getReplicaPort(Replica replica) { String replicaNode = replica.getNodeName(); String tmp = replicaNode.substring(replicaNode.indexOf(':')+1); if (tmp.indexOf('_') != -1) tmp = tmp.substring(0,tmp.indexOf('_')); return Integer.parseInt(tmp); }
if (!zkStateReader.getClusterState().liveNodesContain(pullReplica.getNodeName())) { continue; long replicaIndexVersion = getIndexVersion(pullReplica); if (leaderIndexVersion == replicaIndexVersion) { log.info("Leader replica's version ({}) in sync with replica({}): {} == {}", leader.getName(), pullReplica.getName(), leaderIndexVersion, replicaIndexVersion); try (SolrCore core = containers.get(pullReplica.getNodeName()).getCore(pullReplica.getCoreName())) { RefCounted<SolrIndexSearcher> ref = core.getRegisteredSearcher(); try { break; } else { log.info("Replica {} has the correct version replicated, but the searcher is not ready yet. Replicated version: {}, Serving version: {}", pullReplica.getName(), replicaIndexVersion, servingVersion); if (timeout.hasTimedOut()) { logReplicaTypesReplicationInfo(collectionName, zkStateReader); fail(String.format(Locale.ROOT, "Timed out waiting for replica %s (%d) to replicate from leader %s (%d)", pullReplica.getName(), replicaIndexVersion, leader.getName(), leaderIndexVersion)); log.info("{} version is {} and leader's is {}, will wait for replication", pullReplica.getName(), replicaIndexVersion, leaderIndexVersion); } else { log.info("Leader replica's version ({}) is lower than pull replica({}): {} < {}", leader.getName(), pullReplica.getName(), leaderIndexVersion, replicaIndexVersion);
while (replicaIterator.hasNext()) { Replica replica = replicaIterator.next(); if (!replica.getNodeName().equals(leader.getNodeName()) && !replica.getName().equals(leader.getName())) { ZkCoreNodeProps zkProps1 = new ZkCoreNodeProps(replica); String url1 = zkProps1.getCoreUrl();
protected static void assertAllActive(String collection, ZkStateReader zkStateReader) throws KeeperException, InterruptedException { zkStateReader.forceUpdateCollection(collection); ClusterState clusterState = zkStateReader.getClusterState(); final DocCollection docCollection = clusterState.getCollectionOrNull(collection); if (docCollection == null || docCollection.getSlices() == null) { throw new IllegalArgumentException("Cannot find collection:" + collection); } Map<String,Slice> slices = docCollection.getSlicesMap(); for (Map.Entry<String,Slice> entry : slices.entrySet()) { Slice slice = entry.getValue(); if (slice.getState() != Slice.State.ACTIVE) { fail("Not all shards are ACTIVE - found a shard " + slice.getName() + " that is: " + slice.getState()); } Map<String,Replica> shards = slice.getReplicasMap(); for (Map.Entry<String,Replica> shard : shards.entrySet()) { Replica replica = shard.getValue(); if (replica.getState() != Replica.State.ACTIVE) { fail("Not all replicas are ACTIVE - found a replica " + replica.getName() + " that is: " + replica.getState()); } } } }
protected void logReplicationDetails(Replica replica, StringBuilder builder) throws IOException { try (HttpSolrClient client = new HttpSolrClient.Builder(replica.getCoreUrl()).build()) { ModifiableSolrParams params = new ModifiableSolrParams(); params.set("qt", "/replication"); params.set(ReplicationHandler.COMMAND, ReplicationHandler.CMD_DETAILS); try { QueryResponse response = client.query(params); builder.append(String.format(Locale.ROOT, "%s: %s%s", replica.getName(), response.getResponse(), System.lineSeparator())); } catch (SolrServerException e) { log.warn("Unable to ger replication details for replica {}", replica.getName(), e); } } }
/** * Get the shardId of a core on a specific node */ public String getShardId(String nodeName, String coreName) { for (Slice slice : this) { for (Replica replica : slice) { if (Objects.equals(replica.getNodeName(), nodeName) && Objects.equals(replica.getCoreName(), coreName)) return slice.getName(); } } return null; }
protected void logReplicaTypesReplicationInfo(String collectionName, ZkStateReader zkStateReader) throws KeeperException, InterruptedException, IOException { log.info("## Collecting extra Replica.Type information of the cluster"); zkStateReader.updateLiveNodes(); StringBuilder builder = new StringBuilder(); zkStateReader.forceUpdateCollection(collectionName); DocCollection collection = zkStateReader.getClusterState().getCollection(collectionName); for(Slice s:collection.getSlices()) { Replica leader = s.getLeader(); for (Replica r:s.getReplicas()) { if (!r.isActive(zkStateReader.getClusterState().getLiveNodes())) { builder.append(String.format(Locale.ROOT, "Replica %s not in liveNodes or is not active%s", r.getName(), System.lineSeparator())); continue; } if (r.equals(leader)) { builder.append(String.format(Locale.ROOT, "Replica %s is leader%s", r.getName(), System.lineSeparator())); } logReplicationDetails(r, builder); } } log.info("Summary of the cluster: " + builder.toString()); }