public static List<SolrClient> createHttpSolrClients(Map<String, String> connectionParams, HttpClient httpClient) { List<SolrClient> result = Lists.newArrayList(); for (String shard : SolrConnectionParamUtil.getShards(connectionParams)) { result.add(new HttpSolrClient.Builder(shard).withHttpClient(httpClient).build()); } if (result.size() == 0) { throw new RuntimeException( String.format("You need to specify at least one solr shard connection parameter (%s0={url})", SolrConnectionParams.SOLR_SHARD_PREFIX)); } return result; }
private DirectSolrClassicInputDocumentWriter createClassicSolrWriter(Context context, Map<String, String> indexConnectionParams) throws IOException { PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(); connectionManager.setDefaultMaxPerRoute(getSolrMaxConnectionsPerRoute(indexConnectionParams)); connectionManager.setMaxTotal(getSolrMaxConnectionsTotal(indexConnectionParams)); HttpClient httpClient = new DefaultHttpClient(connectionManager); List<SolrClient> solrServers = createHttpSolrClients(indexConnectionParams, httpClient); return new DirectSolrClassicInputDocumentWriter( context.getConfiguration().get(INDEX_NAME_CONF_KEY), solrServers); }
private Set<SolrClient> createSolrClients(Map<String, String> indexConnectionParams) throws MalformedURLException { String solrMode = getSolrMode(indexConnectionParams); if (solrMode.equals("cloud")) { String indexZkHost = indexConnectionParams.get(SolrConnectionParams.ZOOKEEPER); String collectionName = indexConnectionParams.get(SolrConnectionParams.COLLECTION); CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(indexZkHost).build(); int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(getConf()); solrServer.setZkClientTimeout(zkSessionTimeout); solrServer.setZkConnectTimeout(zkSessionTimeout); solrServer.setDefaultCollection(collectionName); return Collections.singleton((SolrClient)solrServer); } else if (solrMode.equals("classic")) { PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(); connectionManager.setDefaultMaxPerRoute(getSolrMaxConnectionsPerRoute(indexConnectionParams)); connectionManager.setMaxTotal(getSolrMaxConnectionsTotal(indexConnectionParams)); HttpClient httpClient = new DefaultHttpClient(connectionManager); return new HashSet<SolrClient>(createHttpSolrClients(indexConnectionParams, httpClient)); } else { throw new RuntimeException("Only 'cloud' and 'classic' are valid values for solr.mode, but got " + solrMode); } }
private Indexer createIndexer(String indexName, Context context, IndexerConf indexerConf, String tableName, ResultToSolrMapper mapper, Map<String, String> indexConnectionParams) throws IOException, SharderException { Configuration conf = context.getConfiguration(); if (conf.getBoolean(INDEX_DIRECT_WRITE_CONF_KEY, false)) { String solrMode = getSolrMode(indexConnectionParams); if (solrMode.equals("cloud")) { DirectSolrInputDocumentWriter writer = createCloudSolrWriter(context, indexConnectionParams); solrDocWriter = wrapInBufferedWriter(context, writer); return Indexer.createIndexer(indexName, indexerConf, tableName, mapper, null, null, solrDocWriter); } else if (solrMode.equals("classic")) { DirectSolrClassicInputDocumentWriter classicSolrWriter = createClassicSolrWriter(context, indexConnectionParams); Sharder sharder = createSharder(indexConnectionParams, classicSolrWriter.getNumServers()); solrDocWriter = wrapInBufferedWriter(context, classicSolrWriter); return Indexer.createIndexer(indexName, indexerConf, tableName, mapper, null, sharder, solrDocWriter); } else { throw new RuntimeException("Only 'cloud' and 'classic' are valid values for solr.mode, but got " + solrMode); } } else { solrDocWriter = new MapReduceSolrInputDocumentWriter(context); return Indexer.createIndexer(indexName, indexerConf, tableName, mapper, null, null, solrDocWriter); } }
private Set<SolrClient> createSolrServers(Map<String, String> indexConnectionParams, String uniqueKeyField) throws MalformedURLException { String solrMode = getSolrMode(indexConnectionParams); if (solrMode.equals("cloud")) { String indexZkHost = indexConnectionParams.get(SolrConnectionParams.ZOOKEEPER); String collectionName = indexConnectionParams.get(SolrConnectionParams.COLLECTION); CloudSolrClient solrServer = new CloudSolrClient.Builder().withZkHost(indexZkHost).build(); int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(getConf()); solrServer.setZkClientTimeout(zkSessionTimeout); solrServer.setZkConnectTimeout(zkSessionTimeout); solrServer.setDefaultCollection(collectionName); solrServer.setIdField(uniqueKeyField); return Collections.singleton((SolrClient)solrServer); } else if (solrMode.equals("classic")) { PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(); connectionManager.setDefaultMaxPerRoute(getSolrMaxConnectionsPerRoute(indexConnectionParams)); connectionManager.setMaxTotal(getSolrMaxConnectionsTotal(indexConnectionParams)); HttpClient httpClient = new DefaultHttpClient(connectionManager); return new HashSet<SolrClient>(createHttpSolrClients(indexConnectionParams, httpClient)); } else { throw new RuntimeException("Only 'cloud' and 'classic' are valid values for solr.mode, but got " + solrMode); } }
private Indexer createIndexer(String indexName, Context context, IndexerConf indexerConf, String tableName, ResultToSolrMapper mapper, Map<String, String> indexConnectionParams) throws IOException, SharderException { Configuration conf = context.getConfiguration(); if (conf.getBoolean(INDEX_DIRECT_WRITE_CONF_KEY, false)) { String solrMode = getSolrMode(indexConnectionParams); if (solrMode.equals("cloud")) { DirectSolrInputDocumentWriter writer = createCloudSolrWriter( context, indexConnectionParams, indexerConf.getUniqueKeyField()); solrDocWriter = wrapInBufferedWriter(context, writer); return Indexer.createIndexer(indexName, indexerConf, tableName, mapper, null, null, solrDocWriter); } else if (solrMode.equals("classic")) { DirectSolrClassicInputDocumentWriter classicSolrWriter = createClassicSolrWriter(context, indexConnectionParams); Sharder sharder = createSharder(indexConnectionParams, classicSolrWriter.getNumServers()); solrDocWriter = wrapInBufferedWriter(context, classicSolrWriter); return Indexer.createIndexer(indexName, indexerConf, tableName, mapper, null, sharder, solrDocWriter); } else { throw new RuntimeException("Only 'cloud' and 'classic' are valid values for solr.mode, but got " + solrMode); } } else { solrDocWriter = new MapReduceSolrInputDocumentWriter(context); return Indexer.createIndexer(indexName, indexerConf, tableName, mapper, null, null, solrDocWriter); } }
String solrMode = SolrConnectionParamUtil.getSolrMode(connectionParams); if (solrMode.equals("cloud")) { int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(hbaseConf); } else if (solrMode.equals("classic")) { connectionManager = new PoolingClientConnectionManager(); connectionManager.setDefaultMaxPerRoute(getSolrMaxConnectionsPerRoute(connectionParams)); connectionManager.setMaxTotal(getSolrMaxConnectionsTotal(connectionParams));
public static List<SolrClient> createHttpSolrClients(Map<String, String> connectionParams, HttpClient httpClient) { List<SolrClient> result = Lists.newArrayList(); for (String shard : SolrConnectionParamUtil.getShards(connectionParams)) { result.add(new HttpSolrClient.Builder(shard).withHttpClient(httpClient).build()); } if (result.size() == 0) { throw new RuntimeException( String.format("You need to specify at least one solr shard connection parameter (%s0={url})", SolrConnectionParams.SOLR_SHARD_PREFIX)); } return result; }
private DirectSolrClassicInputDocumentWriter createClassicSolrWriter(Context context, Map<String, String> indexConnectionParams) throws IOException { PoolingClientConnectionManager connectionManager = new PoolingClientConnectionManager(); connectionManager.setDefaultMaxPerRoute(getSolrMaxConnectionsPerRoute(indexConnectionParams)); connectionManager.setMaxTotal(getSolrMaxConnectionsTotal(indexConnectionParams)); HttpClient httpClient = new DefaultHttpClient(connectionManager); List<SolrClient> solrServers = createHttpSolrClients(indexConnectionParams, httpClient); return new DirectSolrClassicInputDocumentWriter( context.getConfiguration().get(INDEX_NAME_CONF_KEY), solrServers); }
String solrMode = SolrConnectionParamUtil.getSolrMode(connectionParams); if (solrMode.equals("cloud")) { int zkSessionTimeout = HBaseIndexerConfiguration.getSessionTimeout(hbaseConf); } else if (solrMode.equals("classic")) { connectionManager = new PoolingClientConnectionManager(); connectionManager.setDefaultMaxPerRoute(getSolrMaxConnectionsPerRoute(connectionParams)); connectionManager.setMaxTotal(getSolrMaxConnectionsTotal(connectionParams));
private String[] createBatchArguments(IndexerDefinition indexer) { String[] batchIndexArguments = indexer.getBatchIndexCliArgumentsOrDefault(); List<String> args = Lists.newArrayList(); String mode = Optional.fromNullable(indexer.getConnectionParams().get(SolrConnectionParams.MODE)).or("cloud").toLowerCase(); if ("cloud".equals(mode)) { // cloud mode is the default args.add("--zk-host"); args.add(indexer.getConnectionParams().get(SolrConnectionParams.ZOOKEEPER)); } else { for (String shard : SolrConnectionParamUtil.getShards(indexer.getConnectionParams())) { args.add("--shard-url"); args.add(shard); } } args.add("--hbase-indexer-zk"); args.add(zkConnectString); args.add("--hbase-indexer-name"); args.add(indexer.getName()); args.add("--reducers"); args.add("0"); // additional arguments that were configured on the index (e.g. HBase scan parameters) args.addAll(Lists.newArrayList(batchIndexArguments)); return args.toArray(new String[args.size()]); }
private String[] createBatchArguments(IndexerDefinition indexer) { String[] batchIndexArguments = indexer.getBatchIndexCliArgumentsOrDefault(); List<String> args = Lists.newArrayList(); String mode = Optional.fromNullable(indexer.getConnectionParams().get(SolrConnectionParams.MODE)).or("cloud").toLowerCase(); if ("cloud".equals(mode)) { // cloud mode is the default args.add("--zk-host"); args.add(indexer.getConnectionParams().get(SolrConnectionParams.ZOOKEEPER)); } else { for (String shard : SolrConnectionParamUtil.getShards(indexer.getConnectionParams())) { args.add("--shard-url"); args.add(shard); } } args.add("--hbase-indexer-zk"); args.add(zkConnectString); args.add("--hbase-indexer-name"); args.add(indexer.getName()); args.add("--reducers"); args.add("0"); // additional arguments that were configured on the index (e.g. HBase scan parameters) args.addAll(Lists.newArrayList(batchIndexArguments)); return args.toArray(new String[args.size()]); }
if (SolrConnectionParamUtil.getShards(connectionParams).size() == 0) { throw new CliException("ERROR: You need at least one shard when using solr classic");
if (SolrConnectionParamUtil.getShards(connectionParams).size() == 0) { throw new CliException("ERROR: You need at least one shard when using solr classic");