public SolrZkClient getZkClient() { return solrClient.getZkStateReader().getZkClient(); }
public Map getClusterProps(){ Map result = null; try { if(getZkClient().exists(ZkStateReader.CLUSTER_PROPS, true)){ result = (Map) Utils.fromJSON(getZkClient().getData(ZkStateReader.CLUSTER_PROPS, null, new Stat(), true)) ; } else { result= new LinkedHashMap(); } return result; } catch (Exception e) { throw new SolrException(ErrorCode.SERVER_ERROR,"Error reading cluster properties",e) ; } }
@Override public Map getZkJson(String path) throws KeeperException, InterruptedException { return Utils.getJson(zkClientClusterStateProvider.getZkStateReader().getZkClient(), path, true); }
public SolrClientCloudManager(DistributedQueueFactory queueFactory, CloudSolrClient solrClient) { this.queueFactory = queueFactory; this.solrClient = solrClient; this.zkStateReader = solrClient.getZkStateReader(); this.zkClient = zkStateReader.getZkClient(); this.stateManager = new ZkDistribStateManager(zkClient); this.isClosed = false; this.objectCache = new ObjectCache(); }
@Override public void process(WatchedEvent event) { // session events are not change events, and do not remove the watcher if (EventType.None.equals(event.getType())) { return; } try { synchronized (ZkStateReader.this.getUpdateLock()) { log.debug("Updating [{}] ... ", SOLR_SECURITY_CONF_PATH); // remake watch final Watcher thisWatch = this; final Stat stat = new Stat(); final byte[] data = getZkClient().getData(SOLR_SECURITY_CONF_PATH, thisWatch, stat, true); try { callback.call(new Pair<>(data, stat)); } catch (Exception e) { log.error("Error running collections node listener", e); } } } catch (KeeperException.ConnectionLossException | KeeperException.SessionExpiredException e) { log.warn("ZooKeeper watch triggered, but Solr cannot talk to ZK: [{}]", e.getMessage()); } catch (KeeperException e) { log.error("A ZK error has occurred", e); throw new ZooKeeperException(ErrorCode.SERVER_ERROR, "", e); } catch (InterruptedException e) { // Restore the interrupted status Thread.currentThread().interrupt(); log.warn("Interrupted", e); } }
public static void waitForCollectionToDisappear(String collection, ZkStateReader zkStateReader, boolean verbose, boolean failOnTimeout, int timeoutSeconds) throws Exception { log.info("Wait for collection to disappear - collection: " + collection + " failOnTimeout:" + failOnTimeout + " timeout (sec):" + timeoutSeconds); boolean cont = true; int cnt = 0; while (cont) { if (verbose) System.out.println("-"); ClusterState clusterState = zkStateReader.getClusterState(); if (!clusterState.hasCollection(collection)) break; if (cnt == timeoutSeconds) { if (verbose) System.out.println("Gave up waiting for "+collection+" to disappear.."); if (failOnTimeout) { Diagnostics.logThreadDumps("Gave up waiting for "+collection+" to disappear. THREAD DUMP:"); zkStateReader.getZkClient().printLayoutToStdOut(); fail("The collection ("+collection+") is still present - waited for " + timeoutSeconds + " seconds"); // won't get here return; } cont = false; } else { Thread.sleep(1000); } cnt++; } log.info("Collection has disappeared - collection: " + collection); }
public static DocCollection getCollectionLive(ZkStateReader zkStateReader, String coll) { String collectionPath = getCollectionPath(coll); try { Stat stat = new Stat(); byte[] data = zkStateReader.getZkClient().getData(collectionPath, null, stat, true); ClusterState state = ClusterState.load(stat.getVersion(), data, Collections.<String> emptySet(), collectionPath); ClusterState.CollectionRef collectionRef = state.getCollectionStates().get(coll); return collectionRef == null ? null : collectionRef.get(); } catch (KeeperException.NoNodeException e) { log.warn("No node available : " + collectionPath, e); return null; } catch (KeeperException e) { throw new SolrException(ErrorCode.BAD_REQUEST, "Could not load collection from ZK:" + coll, e); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new SolrException(ErrorCode.BAD_REQUEST, "Could not load collection from ZK:" + coll, e); } }
@Override public void distribSetUp() throws Exception { super.distribSetUp(); // ignoreException(".*"); if (sliceCount > 0) { System.setProperty("numShards", Integer.toString(sliceCount)); } else { System.clearProperty("numShards"); } if (isSSLMode()) { System.clearProperty("urlScheme"); try (ZkStateReader zkStateReader = new ZkStateReader(zkServer.getZkAddress(), AbstractZkTestCase.TIMEOUT, AbstractZkTestCase.TIMEOUT)) { try { zkStateReader.getZkClient().create(ZkStateReader.CLUSTER_PROPS, Utils.toJSON(Collections.singletonMap("urlScheme", "https")), CreateMode.PERSISTENT, true); } catch (KeeperException.NodeExistsException e) { ZkNodeProps props = ZkNodeProps.load(zkStateReader.getZkClient().getData(ZkStateReader.CLUSTER_PROPS, null, null, true)); zkStateReader.getZkClient().setData(ZkStateReader.CLUSTER_PROPS, Utils.toJSON(props.plus("urlScheme", "https")), true); } } } if (useTlogReplicas()) { log.info("Will use {} replicas unless explicitly asked otherwise", Replica.Type.TLOG); } else { log.info("Will use {} replicas unless explicitly asked otherwise", Replica.Type.NRT); } }
/** * Returns the content of /security.json from ZooKeeper as a Map * If the files doesn't exist, it returns null. */ public ConfigData getSecurityProps(boolean getFresh) { if (!getFresh) { if (securityData == null) return new ConfigData(EMPTY_MAP,-1); return new ConfigData(securityData.data, securityData.version); } try { Stat stat = new Stat(); if(getZkClient().exists(SOLR_SECURITY_CONF_PATH, true)) { final byte[] data = getZkClient().getData(ZkStateReader.SOLR_SECURITY_CONF_PATH, null, stat, true); return data != null && data.length > 0 ? new ConfigData((Map<String, Object>) Utils.fromJSON(data), stat.getVersion()) : null; } } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new SolrException(ErrorCode.SERVER_ERROR,"Error reading security properties", e) ; } catch (KeeperException e) { throw new SolrException(ErrorCode.SERVER_ERROR,"Error reading security properties", e) ; } return null; }
/** * Configure, run and return the {@link MiniSolrCloudCluster} * @throws Exception if an error occurs on startup */ public MiniSolrCloudCluster build() throws Exception { MiniSolrCloudCluster cluster = new MiniSolrCloudCluster(nodeCount, baseDir, solrxml, jettyConfig, null, securityJson); CloudSolrClient client = cluster.getSolrClient(); for (Config config : configs) { ((ZkClientClusterStateProvider)client.getClusterStateProvider()).uploadConfig(config.path, config.name); } if (clusterProperties.size() > 0) { ClusterProperties props = new ClusterProperties(cluster.getSolrClient().getZkStateReader().getZkClient()); for (Map.Entry<String, String> entry : clusterProperties.entrySet()) { props.setClusterProperty(entry.getKey(), entry.getValue()); } } return cluster; }
/** * Returns the content of /security.json from ZooKeeper as a Map * If the files doesn't exist, it returns null. */ public ConfigData getSecurityProps(boolean getFresh) { if (!getFresh) { if (securityData == null) return new ConfigData(EMPTY_MAP, -1); return new ConfigData(securityData.data, securityData.version); } try { Stat stat = new Stat(); if(getZkClient().exists(SOLR_SECURITY_CONF_PATH, true)) { byte[] data = getZkClient() .getData(ZkStateReader.SOLR_SECURITY_CONF_PATH, null, stat, true); return data != null && data.length > 0 ? new ConfigData((Map<String, Object>) Utils.fromJSON(data), stat.getVersion()) : null; } } catch (KeeperException | InterruptedException e) { throw new SolrException(ErrorCode.SERVER_ERROR,"Error reading security properties",e) ; } return null; } /**
if (failOnTimeout) { Diagnostics.logThreadDumps("Gave up waiting for recovery to finish. THREAD DUMP:"); zkStateReader.getZkClient().printLayoutToStdOut(); fail("There are still nodes recoverying - waited for " + timeoutSeconds + " seconds");
/** Delete all collections (and aliases) */ public void deleteAllCollections() throws Exception { try (ZkStateReader reader = new ZkStateReader(solrClient.getZkStateReader().getZkClient())) { reader.createClusterStateWatchersAndUpdate(); // up to date aliases & collections reader.aliasesManager.applyModificationAndExportToZk(aliases -> Aliases.EMPTY); for (String collection : reader.getClusterState().getCollectionStates().keySet()) { CollectionAdminRequest.deleteCollection(collection).process(solrClient); } } }
protected static SolrZkClient zkClient() { ZkStateReader reader = cluster.getSolrClient().getZkStateReader(); if (reader == null) cluster.getSolrClient().connect(); return cluster.getSolrClient().getZkStateReader().getZkClient(); }
private static void createCollection(CloudSolrClient cloudSolrClient, String collection, Path configPath) { try { LOGGER.info("Creating SOLR collection {}", collection); int numShards = getEnvInt("SOLR_NUM_SHARDS", 1); int numReplicas = getEnvInt("SOLR_REPLICATION_FACTOR", 1); cloudSolrClient.getZkStateReader().getZkClient().upConfig(configPath, collection); Create createCollection = CollectionAdminRequest.createCollection(collection, collection, numShards, numReplicas); createCollection.setMaxShardsPerNode(getEnvInt("SOLR_MAX_SHARDS_PER_NODE", 1)); createCollection.setAutoAddReplicas(getEnvBoolean("SOLR_AUTO_ADD_REPLICAS", false)); CollectionAdminResponse response = createCollection.process(cloudSolrClient); if (!response.isSuccess()) { LOGGER.error("Could not create collection {}: {}", collection, response.getErrorMessages()); } } catch (SolrServerException | SolrException | IOException e) { LOGGER.error("Error creating collection {}", collection, e); } }
protected void assertCollectionNotExists(String collectionName, int timeoutSeconds) throws Exception { waitForCollectionToDisappear(collectionName, getCommonCloudSolrClient().getZkStateReader(), false, true, timeoutSeconds); assertFalse(cloudClient.getZkStateReader().getZkClient().exists(ZkStateReader.COLLECTIONS_ZKNODE + "/" + collectionName, true)); }
private void createCollectionIfNeeded(CloudSolrClient cloudSolrServer) throws SolrServerException { String solrCollection = remoteSolrServerConfiguration.getSolrCollection(); ZkStateReader zkStateReader = cloudSolrServer.getZkStateReader(); SolrZkClient zkClient = zkStateReader.getZkClient(); log.debug("creating {} collection if needed", solrCollection); try {
private void createCollectionIfNeeded(CloudSolrClient cloudSolrServer) throws SolrServerException { String solrCollection = remoteSolrServerConfiguration.getSolrCollection(); ZkStateReader zkStateReader = cloudSolrServer.getZkStateReader(); SolrZkClient zkClient = zkStateReader.getZkClient(); log.debug("creating {} collection if needed", solrCollection); try {
static void waitForNewLeader(CloudSolrClient cloudClient, String shardName, Replica oldLeader, TimeOut timeOut) throws Exception { log.info("Will wait for a node to become leader for {} secs", timeOut.timeLeft(SECONDS)); ZkStateReader zkStateReader = cloudClient.getZkStateReader(); zkStateReader.forceUpdateCollection(DEFAULT_COLLECTION); for (; ; ) { ClusterState clusterState = zkStateReader.getClusterState(); DocCollection coll = clusterState.getCollection("collection1"); Slice slice = coll.getSlice(shardName); if (slice.getLeader() != null && !slice.getLeader().equals(oldLeader) && slice.getLeader().getState() == Replica.State.ACTIVE) { log.info("Old leader {}, new leader {}. New leader got elected in {} ms", oldLeader, slice.getLeader(),timeOut.timeElapsed(MILLISECONDS) ); break; } if (timeOut.hasTimedOut()) { Diagnostics.logThreadDumps("Could not find new leader in specified timeout"); zkStateReader.getZkClient().printLayoutToStdOut(); fail("Could not find new leader even after waiting for " + timeOut.timeElapsed(MILLISECONDS) + "ms"); } Thread.sleep(100); } }
ZkStateReader zkStateReader = cluster.getSolrClient().getZkStateReader(); zkStateReader .getZkClient() .setData("/security.json", securityJson.getBytes(Charset.defaultCharset()), true); String zkAddress = cluster.getZkServer().getZkAddress();