protected int getReplicaPort(Replica replica) { String replicaNode = replica.getNodeName(); String tmp = replicaNode.substring(replicaNode.indexOf(':')+1); if (tmp.indexOf('_') != -1) tmp = tmp.substring(0,tmp.indexOf('_')); return Integer.parseInt(tmp); }
private void addNodeNameReplica(Replica replica) { List<Replica> replicas = nodeNameReplicas.get(replica.getNodeName()); if (replicas == null) { replicas = new ArrayList<>(); nodeNameReplicas.put(replica.getNodeName(), replicas); } replicas.add(replica); if (replica.getStr(Slice.LEADER) != null) { List<Replica> leaderReplicas = nodeNameLeaderReplicas.get(replica.getNodeName()); if (leaderReplicas == null) { leaderReplicas = new ArrayList<>(); nodeNameLeaderReplicas.put(replica.getNodeName(), leaderReplicas); } leaderReplicas.add(replica); } }
/** * Get the shardId of a core on a specific node */ public String getShardId(String nodeName, String coreName) { for (Slice slice : this) { for (Replica replica : slice) { if (Objects.equals(replica.getNodeName(), nodeName) && Objects.equals(replica.getCoreName(), coreName)) return slice.getName(); } } return null; }
public ReplicaInfo(String coll, String shard, Replica r, Map<String, Object> vals) { this.name = r.getName(); this.core = r.getCoreName(); this.collection = coll; this.shard = shard; this.type = r.getType(); boolean maybeLeader = r.getBool(LEADER_PROP, false); if (vals != null) { this.variables.putAll(vals); maybeLeader = "true".equals(String.valueOf(vals.getOrDefault(LEADER_PROP, maybeLeader))); } this.isLeader = maybeLeader; this.node = r.getNodeName(); }
public Replica getLeader(String collection, String shard) throws InterruptedException { if (clusterState != null) { Replica replica = clusterState.getLeader(collection, shard); if (replica != null && getClusterState().liveNodesContain(replica.getNodeName())) { return replica; } } return null; }
protected void readReplicaDetails() throws IOException { ClusterStateProvider clusterStateProvider = getClusterStateProvider(); ClusterState clusterState = clusterStateProvider.getClusterState(); if (clusterState == null) { // zkStateReader still initializing return; } Map<String, ClusterState.CollectionRef> all = clusterStateProvider.getClusterState().getCollectionStates(); all.forEach((collName, ref) -> { DocCollection coll = ref.get(); if (coll == null) return; if (coll.getProperties().get(CollectionAdminParams.WITH_COLLECTION) != null) { withCollectionsMap.put(coll.getName(), (String) coll.getProperties().get(CollectionAdminParams.WITH_COLLECTION)); } coll.forEachReplica((shard, replica) -> { Map<String, Map<String, List<ReplicaInfo>>> nodeData = nodeVsCollectionVsShardVsReplicaInfo.computeIfAbsent(replica.getNodeName(), k -> new HashMap<>()); Map<String, List<ReplicaInfo>> collData = nodeData.computeIfAbsent(collName, k -> new HashMap<>()); List<ReplicaInfo> replicas = collData.computeIfAbsent(shard, k -> new ArrayList<>()); replicas.add(new ReplicaInfo(collName, shard, replica, new HashMap<>(replica.getProperties()))); }); }); }
protected List<String> getShardUrls() throws IOException { try { ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader(); Slice[] slices = CloudSolrStream.getSlices(this.collection, zkStateReader, false); ClusterState clusterState = zkStateReader.getClusterState(); Set<String> liveNodes = clusterState.getLiveNodes(); List<String> baseUrls = new ArrayList<>(); for(Slice slice : slices) { Collection<Replica> replicas = slice.getReplicas(); List<Replica> shuffler = new ArrayList<>(); for(Replica replica : replicas) { if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) { shuffler.add(replica); } } Collections.shuffle(shuffler, new Random()); Replica rep = shuffler.get(0); ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep); String url = zkProps.getCoreUrl(); baseUrls.add(url); } return baseUrls; } catch (Exception e) { throw new IOException(e); } }
while (replicaIterator.hasNext()) { Replica replica = replicaIterator.next(); if (!replica.getNodeName().equals(leader.getNodeName()) && !replica.getName().equals(leader.getName())) { ZkCoreNodeProps zkProps1 = new ZkCoreNodeProps(replica);
private List<String> getShardUrls() throws IOException { try { ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader(); Slice[] slices = CloudSolrStream.getSlices(this.collection, zkStateReader, false); ClusterState clusterState = zkStateReader.getClusterState(); Set<String> liveNodes = clusterState.getLiveNodes(); List<String> baseUrls = new ArrayList<>(); for(Slice slice : slices) { Collection<Replica> replicas = slice.getReplicas(); List<Replica> shuffler = new ArrayList<>(); for(Replica replica : replicas) { if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) { shuffler.add(replica); } } Collections.shuffle(shuffler, new Random()); Replica rep = shuffler.get(0); ZkCoreNodeProps zkProps = new ZkCoreNodeProps(rep); String url = zkProps.getCoreUrl(); baseUrls.add(url); } return baseUrls; } catch (Exception e) { throw new IOException(e); } }
public Replica getLeader(String collection, String shard) { if (clusterState != null) { DocCollection docCollection = clusterState.getCollectionOrNull(collection); Replica replica = docCollection != null ? docCollection.getLeader(shard) : null; if (replica != null && getClusterState().liveNodesContain(replica.getNodeName())) { return replica; } } return null; }
.getNodeName()); if (live) {
private static void logCollectionStateSummary(String collectionName, ZkStateReader zkStateReader) { Pattern portPattern = Pattern.compile(".*:([0-9]*).*"); DocCollection docCollection = zkStateReader.getClusterState().getCollection(collectionName); if (docCollection == null) { monkeyLog("Could not find collection {}", collectionName); } StringBuilder builder = new StringBuilder(); builder.append("Collection status: {"); for (Slice slice:docCollection.getSlices()) { builder.append(slice.getName() + ": {"); for (Replica replica:slice.getReplicas()) { log.info(replica.toString()); java.util.regex.Matcher m = portPattern.matcher(replica.getBaseUrl()); m.find(); String jettyPort = m.group(1); builder.append(String.format(Locale.ROOT, "%s(%s): {state: %s, type: %s, leader: %s, Live: %s}, ", replica.getName(), jettyPort, replica.getState(), replica.getType(), (replica.get("leader")!= null), zkStateReader.getClusterState().liveNodesContain(replica.getNodeName()))); } if (slice.getReplicas().size() > 0) { builder.setLength(builder.length() - 2); } builder.append("}, "); } if (docCollection.getSlices().size() > 0) { builder.setLength(builder.length() - 2); } builder.append("}"); monkeyLog(builder.toString()); }
for (Replica replica : slices.get(sliceName).getReplicas()) { if (nodesAllowedToRunShards != null && !nodesAllowedToRunShards.contains(replica.getStr(ZkStateReader.NODE_NAME_PROP))) { return "Shard " + replica.getName() + " created on node " + replica.getNodeName() + " not allowed to run shards for the created collection " + collectionName;
List<Replica> shuffler = new ArrayList<>(); for(Replica replica : replicas) { if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) shuffler.add(replica);
if (!directUpdatesToLeadersOnly) { for (Replica replica : slice.getReplicas()) { if (!replica.getNodeName().equals(leader.getNodeName()) && !replica.getName().equals(leader.getName())) { ZkCoreNodeProps zkProps1 = new ZkCoreNodeProps(replica);
private void getPersistedCheckpoints() throws IOException { ZkStateReader zkStateReader = cloudSolrClient.getZkStateReader(); Slice[] slices = CloudSolrStream.getSlices(checkpointCollection, zkStateReader, false); ClusterState clusterState = zkStateReader.getClusterState(); Set<String> liveNodes = clusterState.getLiveNodes(); OUTER: for(Slice slice : slices) { Collection<Replica> replicas = slice.getReplicas(); for(Replica replica : replicas) { if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())){ HttpSolrClient httpClient = streamContext.getSolrClientCache().getHttpSolrClient(replica.getCoreUrl()); try { SolrDocument doc = httpClient.getById(id); if(doc != null) { List<String> checkpoints = (List<String>)doc.getFieldValue("checkpoint_ss"); for (String checkpoint : checkpoints) { String[] pair = checkpoint.split("~"); this.checkpoints.put(pair[0], Long.parseLong(pair[1])); } } } catch (Exception e) { throw new IOException(e); } break OUTER; } } } }
public static int getLiveAndActiveReplicaCount(ZkStateReader zkStateReader, String collection) { Collection<Slice> slices; slices = zkStateReader.getClusterState().getActiveSlices(collection); int liveAndActive = 0; for (Slice slice : slices) { for (Replica replica : slice.getReplicas()) { boolean live = zkStateReader.getClusterState().liveNodesContain(replica.getNodeName()); boolean active = replica.getState() == Replica.State.ACTIVE; if (live && active) { liveAndActive++; } } } return liveAndActive; }
public static int getLiveAndActiveReplicaCount(ZkStateReader zkStateReader, String collection) { Slice[] slices; slices = zkStateReader.getClusterState().getCollection(collection).getActiveSlicesArr(); int liveAndActive = 0; for (Slice slice : slices) { for (Replica replica : slice.getReplicas()) { boolean live = zkStateReader.getClusterState().liveNodesContain(replica.getNodeName()); boolean active = replica.getState() == Replica.State.ACTIVE; if (live && active) { liveAndActive++; } } } return liveAndActive; }
&& clusterState.getLiveNodes().contains(replica.getNodeName())) { randomActiveReplica = replica; break;
params.set("rows", 1); for(Replica replica : replicas) { if(replica.getState() == Replica.State.ACTIVE && liveNodes.contains(replica.getNodeName())) { String coreUrl = replica.getCoreUrl(); SolrStream solrStream = new SolrStream(coreUrl, params);