/** * Initialize any state for this DB. Called once per DB instance; there is one DB instance per * client thread. */ @Override public void init() throws DBException { Properties props = getProperties(); commitTime = Integer .parseInt(props.getProperty("solr.commit.within.time", DEFAULT_COMMIT_WITHIN_TIME)); batchMode = Boolean.parseBoolean(props.getProperty("solr.batch.mode", DEFAULT_BATCH_MODE)); String jaasConfPath = props.getProperty("solr.jaas.conf.path"); if(jaasConfPath != null) { System.setProperty("java.security.auth.login.config", jaasConfPath); HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer()); } // Check if Solr cluster is running in SolrCloud or Stand-alone mode Boolean cloudMode = Boolean.parseBoolean(props.getProperty("solr.cloud", DEFAULT_CLOUD_MODE)); System.err.println("Solr Cloud Mode = " + cloudMode); if (cloudMode) { System.err.println("Solr Zookeeper Remote Hosts = " + props.getProperty("solr.zookeeper.hosts", DEFAULT_ZOOKEEPER_HOSTS)); client = new CloudSolrClient.Builder().withZkHost( Arrays.asList(props.getProperty("solr.zookeeper.hosts", DEFAULT_ZOOKEEPER_HOSTS).split(","))).build(); } else { client = new HttpSolrClient.Builder(props.getProperty("solr.base.url", DEFAULT_SOLR_BASE_URL)).build(); } }
LOG.info("Committing live merge..."); if (options.zkHost != null) { try (CloudSolrClient server = new CloudSolrClient.Builder().withZkHost(options.zkHost).build()) { server.setDefaultCollection(options.collection); server.commit();
LOG.info("Committing live merge..."); if (options.zkHost != null) { try (CloudSolrClient server = new CloudSolrClient.Builder().withZkHost(options.zkHost).build()) { server.setDefaultCollection(options.collection); server.commit();
String uniqueKeyField = indexerConf.getUniqueKeyField(); Preconditions.checkNotNull(uniqueKeyField); CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(hbaseIndexingOpts.zkHost).build(); int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(conf); solrServer.setZkClientTimeout(zkSessionTimeout);
CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(hbaseIndexingOpts.zkHost).build(); int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(conf); solrServer.setZkClientTimeout(zkSessionTimeout);
LOG.info("Committing live merge..."); if (options.zkHost != null) { CloudSolrClient server = new CloudSolrClient.Builder().withZkHost(options.zkHost).build(); server.setDefaultCollection(options.collection); server.commit();
CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder(zkHosts, zkChroot).build();
CloudSolrClient client = new CloudSolrClient.Builder(Arrays.asList(host), Optional.empty()).build(); client.setDefaultCollection(collection);
final CloudSolrClient cloudServer = new CloudSolrClient.Builder() .withLBHttpSolrClientBuilder( new LBHttpSolrClient.Builder() .withHttpSolrClientBuilder(new HttpSolrClient.Builder().withInvariantParams(clientParams)) .withBaseSolrUrls(config.get(HTTP_URLS)) .withZkHost(zookeeperUrls) .sendUpdatesOnlyToShardLeaders() .build(); cloudServer.connect(); solrClient = cloudServer;
new CloudSolrClient.Builder(zkHosts, Optional.ofNullable(chroot)).build();
throw new MorphlineCompilationException("Parameter 'zkHost' requires that you also pass parameter 'collection'", config); CloudSolrClient cloudSolrClient = new Builder() .withZkHost(zkHost) .build(); cloudSolrClient.setDefaultCollection(collectionName); cloudSolrClient.connect();
/** * Instantiate Chronix Client. This should be called by subclasses' @OnScheduled method to create a client * if one does not yet exist. If called when scheduled, closeClient() should be called by the subclasses' @OnStopped * method so the client will be destroyed when the processor is stopped. * * @param context The context for this processor * @throws ProcessException if an error occurs while creating an Chronix client */ protected void createSolrClient(ControllerServiceInitializationContext context) throws ProcessException { if (solr != null) { return; } // create a solr client final boolean isCloud = context.getPropertyValue(SOLR_CLOUD).asBoolean(); final String connectionString = context.getPropertyValue(SOLR_CONNECTION_STRING).asString(); final String collection = context.getPropertyValue(SOLR_COLLECTION).asString(); if (isCloud) { //logInfo("creating solrCloudClient on $solrUrl for collection $collection"); CloudSolrClient cloudSolrClient = new CloudSolrClient.Builder().withZkHost(connectionString).build(); cloudSolrClient.setDefaultCollection(collection); cloudSolrClient.setZkClientTimeout(30000); cloudSolrClient.setZkConnectTimeout(30000); solr = cloudSolrClient; } else { // logInfo(s"creating HttpSolrClient on $solrUrl for collection $collection") solr = new HttpSolrClient.Builder(connectionString + "/" + collection).build(); } }
/** * Creates a SolrCloud collection if it doesn't already exist with the given config set. * * @deprecated since 1.2.0 use {@link SolrCollectionInitializer} instead. * @param zkHost host and optional port of one of a zookeeper instance (localhost:2181) * @param collectionName name of the collection to be created * @param configSetName name of the config set to use * @param configSetPath location of the config set (folder containing solr configs such as * solrconfig.xml) * @throws IOException if something goes wrong with the upload * @throws SolrServerException if something goes wrong with the upload */ @Deprecated public static void initSolr( String zkHost, String collectionName, String configSetName, Path configSetPath) throws IOException, SolrServerException { try (ZkClientClusterStateProvider provider = new ZkClientClusterStateProvider(zkHost)) { provider.uploadConfig(configSetPath, configSetName); } try (SolrClient solr = new CloudSolrClient.Builder().withZkHost(zkHost).build()) { List<String> existingCollectionNames = CollectionAdminRequest.listCollections(solr); if (!existingCollectionNames.contains(collectionName)) { solr.request(CollectionAdminRequest.createCollection(collectionName, configSetName, 1, 1)); } } }
/** * Create a Solr collection with a given number of shards. * * @param collectionName name of the collection to be created * @param configName name of the config for the collection * @param numShards number of shards in the collection */ public void createCollection(String collectionName, String configName, int numShards) throws IOException { int TIMEOUT = 30; // seconds CloudSolrClient solrClient = new CloudSolrClient.Builder().withZkHost(getZkConnectString()).build(); solrClient.setDefaultCollection(collectionName); CollectionAdminRequest.Create request = CollectionAdminRequest.createCollection( collectionName, configName, numShards, 1); request.setMaxShardsPerNode(numShards); RequestStatusState state; try { state = request.processAndWait(solrClient, TIMEOUT); } catch (SolrServerException | InterruptedException e) { throw new RuntimeException(e); } if (state != RequestStatusState.COMPLETED && state != RequestStatusState.RUNNING) { throw new IllegalStateException("Unexpected state: " + state); } waitForRecoveriesToFinish( collectionName, solrClient.getZkStateReader(), true, true, TIMEOUT); }
public SolrClient getSolrServer() { if (zkHost != null && zkHost.length() > 0) { if (collectionName == null || collectionName.length() == 0) { throw new MorphlineCompilationException("Parameter 'zkHost' requires that you also pass parameter 'collection'", config); } CloudSolrClient cloudSolrClient = new Builder() .withZkHost(zkHost) .build(); cloudSolrClient.setDefaultCollection(collectionName); cloudSolrClient.setZkClientTimeout(zkClientSessionTimeout); cloudSolrClient.setZkConnectTimeout(zkClientConnectTimeout); return cloudSolrClient; } else { if (solrUrl == null && solrHomeDir != null) { CoreContainer coreContainer = new CoreContainer(solrHomeDir); coreContainer.load(); EmbeddedSolrServer embeddedSolrServer = new EmbeddedSolrServer(coreContainer, collectionName); return embeddedSolrServer; } if (solrUrl == null || solrUrl.length() == 0) { throw new MorphlineCompilationException("Missing parameter 'solrUrl'", config); } int solrServerNumThreads = 2; int solrServerQueueLength = solrServerNumThreads; SolrClient server = new SafeConcurrentUpdateSolrServer(solrUrl, solrServerQueueLength, solrServerNumThreads); return server; } }
public static SolrClient getClient(Map stormConf, String boltType) { String zkHost = ConfUtils.getString(stormConf, "solr." + boltType + ".zkhost", null); String solrUrl = ConfUtils.getString(stormConf, "solr." + boltType + ".url", null); String collection = ConfUtils.getString(stormConf, "solr." + boltType + ".collection", null); int queueSize = ConfUtils.getInt(stormConf, "solr." + boltType + ".queueSize", -1); SolrClient client; if (StringUtils.isNotBlank(zkHost)) { client = new CloudSolrClient.Builder().withZkHost(zkHost).build(); if (StringUtils.isNotBlank(collection)) { ((CloudSolrClient) client).setDefaultCollection(collection); } } else if (StringUtils.isNotBlank(solrUrl)) { if (queueSize == -1) { client = new HttpSolrClient.Builder(solrUrl).build(); } else { client = new ConcurrentUpdateSolrClient.Builder(solrUrl) .withQueueSize(queueSize).build(); } } else { throw new RuntimeException( "SolrClient should have zk or solr URL set up"); } return client; }
public void open() throws IOException { this.tuples = new TreeSet(); this.solrStreams = new ArrayList(); this.eofTuples = Collections.synchronizedMap(new HashMap()); if(checkpoints.size() == 0 && streamContext.numWorkers > 1) { //Each worker must maintain its own checkpoints this.id = this.id+"_"+streamContext.workerID; } if(streamContext.getSolrClientCache() != null) { cloudSolrClient = streamContext.getSolrClientCache().getCloudSolrClient(zkHost); } else { final List<String> hosts = new ArrayList<String>(); hosts.add(zkHost); cloudSolrClient = new Builder(hosts, Optional.empty()) .build(); this.cloudSolrClient.connect(); } if(checkpoints.size() == 0) { getPersistedCheckpoints(); if(checkpoints.size() == 0) { getCheckpoints(); } } constructStreams(); openStreams(); }
public void open() throws IOException { if(cache != null) { cloudSolrClient = cache.getCloudSolrClient(zkHost); } else { final List<String> hosts = new ArrayList<>(); hosts.add(zkHost); cloudSolrClient = new CloudSolrClient.Builder(hosts, Optional.empty()).build(); } ModifiableSolrParams params = getParams(this.props); params.remove(SORT); //Override any sort. Random rand = new Random(); int seed = rand.nextInt(); String sortField = "random_"+seed; params.add(SORT, sortField+" asc"); QueryRequest request = new QueryRequest(params); try { QueryResponse response = request.process(cloudSolrClient, collection); SolrDocumentList docs = response.getResults(); documentIterator = docs.iterator(); } catch (Exception e) { throw new IOException(e); } }
public void open() throws IOException { if(cache != null) { cloudSolrClient = cache.getCloudSolrClient(zkHost); } else { final List<String> hosts = new ArrayList<>(); hosts.add(zkHost); cloudSolrClient = new Builder(hosts, Optional.empty()).build(); } FieldComparator[] adjustedSorts = adjustSorts(buckets, bucketSorts); this.resortNeeded = resortNeeded(adjustedSorts); String json = getJsonFacetString(buckets, metrics, adjustedSorts, method, refine, bucketSizeLimit); assert expectedJson(json); ModifiableSolrParams paramsLoc = new ModifiableSolrParams(params); paramsLoc.set("json.facet", json); paramsLoc.set("rows", "0"); QueryRequest request = new QueryRequest(paramsLoc); try { NamedList response = cloudSolrClient.request(request, collection); getTuples(response, buckets, metrics); if(resortNeeded) { Collections.sort(tuples, getStreamSort()); } index=this.offset; } catch (Exception e) { throw new IOException(e); } }
private Set<SolrClient> createSolrServers(Map<String, String> indexConnectionParams, String uniqueKeyField) throws MalformedURLException { String solrMode = getSolrMode(indexConnectionParams); if (solrMode.equals("cloud")) { String indexZkHost = indexConnectionParams.get(SolrConnectionParams.ZOOKEEPER); String collectionName = indexConnectionParams.get(SolrConnectionParams.COLLECTION); CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(indexZkHost).build(); int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(getConf()); solrServer.setZkClientTimeout(zkSessionTimeout); solrServer.setZkConnectTimeout(zkSessionTimeout); solrServer.setDefaultCollection(collectionName); solrServer.setIdField(uniqueKeyField); return Collections.singleton((SolrClient)solrServer); } else if (solrMode.equals("classic")) { PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(); connectionManager.setDefaultMaxPerRoute(getSolrMaxConnectionsPerRoute(indexConnectionParams)); connectionManager.setMaxTotal(getSolrMaxConnectionsTotal(indexConnectionParams)); HttpClient httpClient = new DefaultHttpClient(connectionManager); return new HashSet<SolrClient>(createHttpSolrClients(indexConnectionParams, httpClient)); } else { throw new RuntimeException("Only 'cloud' and 'classic' are valid values for solr.mode, but got " + solrMode); } }