/** * Initialize any state for this DB. Called once per DB instance; there is one DB instance per * client thread. */ @Override public void init() throws DBException { Properties props = getProperties(); commitTime = Integer .parseInt(props.getProperty("solr.commit.within.time", DEFAULT_COMMIT_WITHIN_TIME)); batchMode = Boolean.parseBoolean(props.getProperty("solr.batch.mode", DEFAULT_BATCH_MODE)); String jaasConfPath = props.getProperty("solr.jaas.conf.path"); if(jaasConfPath != null) { System.setProperty("java.security.auth.login.config", jaasConfPath); HttpClientUtil.setConfigurer(new Krb5HttpClientConfigurer()); } // Check if Solr cluster is running in SolrCloud or Stand-alone mode Boolean cloudMode = Boolean.parseBoolean(props.getProperty("solr.cloud", DEFAULT_CLOUD_MODE)); System.err.println("Solr Cloud Mode = " + cloudMode); if (cloudMode) { System.err.println("Solr Zookeeper Remote Hosts = " + props.getProperty("solr.zookeeper.hosts", DEFAULT_ZOOKEEPER_HOSTS)); client = new CloudSolrClient.Builder().withZkHost( Arrays.asList(props.getProperty("solr.zookeeper.hosts", DEFAULT_ZOOKEEPER_HOSTS).split(","))).build(); } else { client = new HttpSolrClient.Builder(props.getProperty("solr.base.url", DEFAULT_SOLR_BASE_URL)).build(); } }
public synchronized CloudSolrClient getCloudSolrClient(String zkHost) { CloudSolrClient client; if (solrClients.containsKey(zkHost)) { client = (CloudSolrClient) solrClients.get(zkHost); } else { final List<String> hosts = new ArrayList<String>(); hosts.add(zkHost); CloudSolrClient.Builder builder = new CloudSolrClient.Builder(hosts, Optional.empty()); if (httpClient != null) { builder = builder.withHttpClient(httpClient); } client = builder.build(); client.connect(); solrClients.put(zkHost, client); } return client; }
private void setCloudSolrClient() { if(this.cache != null) { this.cloudSolrClient = this.cache.getCloudSolrClient(zkHost); } else { final List<String> hosts = new ArrayList<>(); hosts.add(zkHost); this.cloudSolrClient = new Builder(hosts, Optional.empty()).build(); this.cloudSolrClient.connect(); } }
public void open() throws IOException { if(cache != null) { cloudSolrClient = cache.getCloudSolrClient(zkHost); } else { final List<String> hosts = new ArrayList<>(); hosts.add(zkHost); cloudSolrClient = new CloudSolrClient.Builder(hosts, Optional.empty()).build(); } ModifiableSolrParams params = getParams(this.props); params.remove(SORT); //Override any sort. Random rand = new Random(); int seed = rand.nextInt(); String sortField = "random_"+seed; params.add(SORT, sortField+" asc"); QueryRequest request = new QueryRequest(params); try { QueryResponse response = request.process(cloudSolrClient, collection); SolrDocumentList docs = response.getResults(); documentIterator = docs.iterator(); } catch (Exception e) { throw new IOException(e); } }
public void open() throws IOException { this.tuples = new TreeSet(); this.solrStreams = new ArrayList(); this.eofTuples = Collections.synchronizedMap(new HashMap()); if(checkpoints.size() == 0 && streamContext.numWorkers > 1) { //Each worker must maintain its own checkpoints this.id = this.id+"_"+streamContext.workerID; } if(streamContext.getSolrClientCache() != null) { cloudSolrClient = streamContext.getSolrClientCache().getCloudSolrClient(zkHost); } else { final List<String> hosts = new ArrayList<String>(); hosts.add(zkHost); cloudSolrClient = new Builder(hosts, Optional.empty()) .build(); this.cloudSolrClient.connect(); } if(checkpoints.size() == 0) { getPersistedCheckpoints(); if(checkpoints.size() == 0) { getCheckpoints(); } } constructStreams(); openStreams(); }
private DirectSolrInputDocumentWriter createCloudSolrWriter( Context context, Map<String, String> indexConnectionParams, String uniqueKeyField) throws IOException { String indexZkHost = indexConnectionParams.get(SolrConnectionParams.ZOOKEEPER); String collectionName = indexConnectionParams.get(SolrConnectionParams.COLLECTION); if (indexZkHost == null) { throw new IllegalStateException("No index ZK host defined"); } if (collectionName == null) { throw new IllegalStateException("No collection name defined"); } CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(indexZkHost).build(); int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(context.getConfiguration()); solrServer.setZkClientTimeout(zkSessionTimeout); solrServer.setZkConnectTimeout(zkSessionTimeout); solrServer.setDefaultCollection(collectionName); solrServer.setIdField(uniqueKeyField); return new DirectSolrInputDocumentWriter(context.getConfiguration().get(INDEX_NAME_CONF_KEY), solrServer); }
public void open() throws IOException { if (cache != null) { cloudSolrClient = cache.getCloudSolrClient(zkHost); } else { final List<String> hosts = new ArrayList<>(); hosts.add(zkHost); cloudSolrClient = new Builder(hosts, Optional.empty()).build(); } String json = getJsonFacetString(field, metrics, start, end, gap); ModifiableSolrParams paramsLoc = new ModifiableSolrParams(params); paramsLoc.set("json.facet", json); paramsLoc.set("rows", "0"); QueryRequest request = new QueryRequest(paramsLoc); try { NamedList response = cloudSolrClient.request(request, collection); getTuples(response, field, metrics); } catch (Exception e) { throw new IOException(e); } }
@Override public SolrClient create() throws Exception { // SolrCloud if (args.solrCloud) { List<String> urls = Splitter.on(',').splitToList(args.solrUrl); if (StringUtils.isNotEmpty(args.solrZkChroot)) { return new CloudSolrClient.Builder(urls, Optional.of(args.solrZkChroot)).build(); // Connect to ZooKeeper } else { return new CloudSolrClient.Builder(urls).build(); // Connect to list of Solr servers } } // Standlone return new ConcurrentUpdateSolrClient.Builder(args.solrUrl) .withQueueSize(args.solrBatch) .withThreadCount(args.threads) .build(); }
/** * Method to initialize a Solr collection using the provided default configs. * * @deprecated since 1.2.0 use {@link SolrCollectionInitializer} instead. * @param zkHost host and optional port of one of a zookeeper instance (localhost:2181) * @param collectionName name of the collection to be created * @throws IOException if something goes wrong with the upload * @throws SolrServerException if something goes wrong with the upload */ @Deprecated public static void initSolr(String zkHost, String collectionName) throws IOException, SolrServerException { try (SolrClient solr = new CloudSolrClient.Builder().withZkHost(zkHost).build()) { List<String> existingCollectionNames = CollectionAdminRequest.listCollections(solr); if (!existingCollectionNames.contains(collectionName)) { solr.request(CollectionAdminRequest.createCollection(collectionName, 1, 1)); } } } }
private Set<SolrClient> createSolrClients(Map<String, String> indexConnectionParams) throws MalformedURLException { String solrMode = getSolrMode(indexConnectionParams); if (solrMode.equals("cloud")) { String indexZkHost = indexConnectionParams.get(SolrConnectionParams.ZOOKEEPER); String collectionName = indexConnectionParams.get(SolrConnectionParams.COLLECTION); CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(indexZkHost).build(); int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(getConf()); solrServer.setZkClientTimeout(zkSessionTimeout); solrServer.setZkConnectTimeout(zkSessionTimeout); solrServer.setDefaultCollection(collectionName); return Collections.singleton((SolrClient)solrServer); } else if (solrMode.equals("classic")) { PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(); connectionManager.setDefaultMaxPerRoute(getSolrMaxConnectionsPerRoute(indexConnectionParams)); connectionManager.setMaxTotal(getSolrMaxConnectionsTotal(indexConnectionParams)); HttpClient httpClient = new DefaultHttpClient(connectionManager); return new HashSet<SolrClient>(createHttpSolrClients(indexConnectionParams, httpClient)); } else { throw new RuntimeException("Only 'cloud' and 'classic' are valid values for solr.mode, but got " + solrMode); } }
public SolrCollectionManager() { // build client to connect to solr SolrFlintConfig config = SolrFlintConfig.getInstance(); // use cloud? Collection<String> zkhosts = config.getZKHosts(); if (zkhosts != null && !zkhosts.isEmpty()) { this.defaultShards = zkhosts.size(); this.defaultReplicas = 1; this._solr = new CloudSolrClient.Builder().withZkHost(zkhosts).build(); } else { this.defaultShards = 1; this.defaultReplicas = 1; this._solr = new HttpSolrClient.Builder(config.getServerURL()).allowCompression(true).build(); } }
private DirectSolrInputDocumentWriter createCloudSolrWriter(Context context, Map<String, String> indexConnectionParams) throws IOException { String indexZkHost = indexConnectionParams.get(SolrConnectionParams.ZOOKEEPER); String collectionName = indexConnectionParams.get(SolrConnectionParams.COLLECTION); if (indexZkHost == null) { throw new IllegalStateException("No index ZK host defined"); } if (collectionName == null) { throw new IllegalStateException("No collection name defined"); } CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(indexZkHost).build(); int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(context.getConfiguration()); solrServer.setZkClientTimeout(zkSessionTimeout); solrServer.setZkConnectTimeout(zkSessionTimeout); solrServer.setDefaultCollection(collectionName); return new DirectSolrInputDocumentWriter(context.getConfiguration().get(INDEX_NAME_CONF_KEY), solrServer); }
protected CloudSolrClient buildSolrClient() { return new Builder(Collections.singletonList(getZkServer().getZkAddress()), Optional.empty()) .build(); }
public static SolrClient createCloudSolrClient(Map<String, String> connectionParameters, String uniqueKeyField, int zkSessionTimeout) { String solrZk = connectionParameters.get(SolrConnectionParams.ZOOKEEPER); CloudSolrClient solr = new CloudSolrClient.Builder().withZkHost(solrZk).build(); solr.setZkClientTimeout(zkSessionTimeout); solr.setZkConnectTimeout(zkSessionTimeout); String collection = connectionParameters.get(SolrConnectionParams.COLLECTION); solr.setDefaultCollection(collection); solr.setIdField(uniqueKeyField); return solr; }
public SolrCollectionManager(Collection<String> zkhosts) { this.defaultShards = zkhosts.size(); this.defaultReplicas = 1; this._solr = new CloudSolrClient.Builder().withZkHost(zkhosts).build(); }
public static SolrClient createCloudSolrClient(Map<String, String> connectionParameters, int zkSessionTimeout) { String solrZk = connectionParameters.get(SolrConnectionParams.ZOOKEEPER); CloudSolrClient solr = new CloudSolrClient.Builder().withZkHost(solrZk).build(); solr.setZkClientTimeout(zkSessionTimeout); solr.setZkConnectTimeout(zkSessionTimeout); String collection = connectionParameters.get(SolrConnectionParams.COLLECTION); solr.setDefaultCollection(collection); return solr; }
public SolrIndexIO(Index index) { this._collection = index.getIndexID(); // build client to connect to solr SolrFlintConfig config = SolrFlintConfig.getInstance(); // use cloud? Collection<String> zkhosts = config.getZKHosts(); if (zkhosts != null && !zkhosts.isEmpty()) { this._client = new CloudSolrClient.Builder().withZkHost(zkhosts).build(); } else { this._client = new HttpSolrClient.Builder(config.getServerURL()).allowCompression(true).build(); } }
private CloudSolrClient getCloudServer() { CloudSolrClient rVal = null; if (this.getZkHost() != null && this.getCollection() != null) { rVal = new CloudSolrClient.Builder().withZkHost(zkHost).build(); rVal.setDefaultCollection(this.getCollection()); } return rVal; }
/** * Builds a Solr client using the ZK hosts specified. * @return SolrClient */ public SolrClient getSolrClient(List<String> zkHosts) { return new CloudSolrClient.Builder().withZkHost(zkHosts).build(); }
/** * Builds a Solr client using the ZK hosts from the global config. * @return SolrClient */ public SolrClient getSolrClient() { return new CloudSolrClient.Builder().withZkHost(getZkHosts()).build(); }