private UpdateRequest newUpdateRequest() { UpdateRequest req = new UpdateRequest(); if(waitSearcher) { req.setAction(UpdateRequest.ACTION.COMMIT, true, true); } return req; }
@Override public SolrRequest toSolrRequest(ITuple tuple) throws SolrMapperException { SolrInputDocument doc = buildDocument(tuple); UpdateRequest request = new UpdateRequest(); request.add(doc); log.debug("Created SolrRequest with content: {}", doc); return request; }
private SolrRequest createtSolrRequest(String json) { final ContentStreamUpdateRequest request = new ContentStreamUpdateRequest(jsonUpdateUrl); final ContentStream cs = new ContentStreamBase.StringStream(json, CONTENT_TYPE); request.addContentStream(cs); if (logger.isDebugEnabled()) { logger.debug("Request generated with JSON: " + json); } return request; }
private void index(boolean isSolrCloud, String collection, Long commitWithin, String contentStreamPath, MultiMapSolrParams requestParams, List<SolrInputDocument> inputDocumentList) throws IOException, SolrServerException,SolrException { UpdateRequest request = new UpdateRequest(contentStreamPath); request.setParams(new ModifiableSolrParams()); // add the extra params, don't use 'set' in case of repeating params Iterator<String> paramNames = requestParams.getParameterNamesIterator(); while (paramNames.hasNext()) { String paramName = paramNames.next(); for (String paramValue : requestParams.getParams(paramName)) { request.getParams().add(paramName, paramValue); } } // specify the collection for SolrCloud if (isSolrCloud) { request.setParam(COLLECTION_PARAM_NAME, collection); } if (commitWithin != null && commitWithin > 0) { request.setParam(COMMIT_WITHIN_PARAM_NAME, commitWithin.toString()); } // if a username and password were provided then pass them for basic auth if (isBasicAuthEnabled()) { request.setBasicAuthCredentials(getUsername(), getPassword()); } request.add(inputDocumentList); UpdateResponse response = request.process(getSolrClient()); getLogger().debug("Got {} response from Solr", new Object[]{response.getStatus()}); inputDocumentList.clear(); }
protected void createCollectionIfNotExist(CloudSolrClient client, String collectionName) { Set<String> collectionNames = client.getZkStateReader().getClusterState().getCollections(); if (!collectionNames.contains(collectionName)) { try { new CollectionAdminRequest.Create().setCollectionName(collectionName).setNumShards(getSolrCloudNumShards()) .setMaxShardsPerNode(getSolrCloudNumShards()).setConfigName(getSolrCloudConfigName()).process(client); } catch (SolrServerException e) { throw ExceptionHelper.refineException(e); } catch (IOException e) { throw ExceptionHelper.refineException(e); } } }
NamedList<Object> listResponse = new CollectionAdminRequest.List().process(primary).getResponse(); List<String> collectionNames = listResponse.get("collections") == null ? collectionNames = new ArrayList<>() : (List<String>) listResponse.get("collections"); new CollectionAdminRequest.Create().setCollectionName(collectionName).setNumShards(solrCloudNumShards) .setConfigName(solrCloudConfigName).process(primary); new CollectionAdminRequest.CreateAlias().setAliasName(primary.getDefaultCollection()) .setAliasedCollections(collectionName).process(primary); } else { collectionName = collectionName.split(",")[0]; if (!collectionNames.contains(collectionName)) { new CollectionAdminRequest.Create().setCollectionName(collectionName).setNumShards(solrCloudNumShards) .setConfigName(solrCloudConfigName).process(primary); listResponse = new CollectionAdminRequest.List().process(primary).getResponse(); collectionNames = listResponse.get("collections") == null ? collectionNames = new ArrayList<>() : (List<String>) listResponse.get("collections"); new CollectionAdminRequest.Create().setCollectionName(collectionName).setNumShards(solrCloudNumShards) .setConfigName(solrCloudConfigName).process(primary); new CollectionAdminRequest.CreateAlias().setAliasName(reindex.getDefaultCollection()) .setAliasedCollections(collectionName).process(primary); } else { collectionName = collectionName.split(",")[0]; if (!collectionNames.contains(collectionName)) { new CollectionAdminRequest.Create().setCollectionName(collectionName).setNumShards(solrCloudNumShards) .setConfigName(solrCloudConfigName).process(primary);
new CollectionAdminRequest.CreateAlias().setAliasName(primaryCloudClient.getDefaultCollection()) .setAliasedCollections(reindexCollectionName).process(primaryCloudClient); new CollectionAdminRequest.CreateAlias().setAliasName(reindexCloudClient.getDefaultCollection()) .setAliasedCollections(primaryCollectionName).process(reindexCloudClient); } catch (Exception e) { LOG.error("An exception occured swapping cores.", e); CoreAdminRequest car = new CoreAdminRequest(); car.setCoreName(primaryCoreName); car.setOtherCoreName(reindexCoreName); car.setAction(CoreAdminAction.SWAP);
private void commitDocumentChanges(String collectionName, Collection<SolrInputDocument> documents) throws SolrServerException, IOException { if (documents.size() == 0) return; try { solrClient.request(newUpdateRequest().add(documents), collectionName); } catch (HttpSolrClient.RemoteSolrException rse) { logger.error("Unable to save documents to Solr as one of the shape objects stored were not compatible with Solr.", rse); logger.error("Details in failed document batch: "); for (SolrInputDocument d : documents) { Collection<String> fieldNames = d.getFieldNames(); for (String name : fieldNames) { logger.error(name + ":" + d.getFieldValue(name).toString()); } } throw rse; } }
@Override public void process(final InputStream in) throws IOException { ContentStreamUpdateRequest request = new ContentStreamUpdateRequest(contentStreamPath); request.setParams(new ModifiableSolrParams()); String paramName = paramNames.next(); for (String paramValue : requestParams.getParams(paramName)) { request.getParams().add(paramName, paramValue); request.setParam(COLLECTION_PARAM_NAME, collection); request.setParam(COMMIT_WITHIN_PARAM_NAME, commitWithin.toString()); request.setBasicAuthCredentials(getUsername(), getPassword()); request.addContentStream(new ContentStreamBase() { @Override public InputStream getStream() throws IOException { UpdateResponse response = request.process(getSolrClient()); getLogger().debug("Got {} response from Solr", new Object[]{response.getStatus()}); } catch (SolrException e) {
private String getFieldNameOfUniqueKey() { final SolrQuery solrQuery = new SolrQuery(); try { solrQuery.setRequestHandler("/schema/uniquekey"); final QueryRequest req = new QueryRequest(solrQuery); if (isBasicAuthEnabled()) { req.setBasicAuthCredentials(getUsername(), getPassword()); } return(req.process(getSolrClient()).getResponse().get("uniqueKey").toString()); } catch (SolrServerException | IOException e) { getLogger().error("Solr query to retrieve uniqueKey-field failed due to {}", new Object[]{solrQuery.toString(), e}, e); throw new ProcessException(e); } }
public QueryRequest getListCollectionsRequest() { ModifiableSolrParams params = new ModifiableSolrParams(); params.set(SolrConstants.REQUEST_ACTION, CollectionParams.CollectionAction.LIST.name()); QueryRequest request = new QueryRequest(params); request.setPath(SolrConstants.REQUEST_COLLECTIONS_PATH); return request; } }
@Override public void clearStorage() throws BackendException { try { if (mode!=Mode.CLOUD) throw new UnsupportedOperationException("Operation only supported for SolrCloud"); logger.debug("Clearing storage from Solr: {}", solrClient); ZkStateReader zkStateReader = ((CloudSolrClient) solrClient).getZkStateReader(); zkStateReader.updateClusterState(true); ClusterState clusterState = zkStateReader.getClusterState(); for (String collection : clusterState.getCollections()) { logger.debug("Clearing collection [{}] in Solr",collection); UpdateRequest deleteAll = newUpdateRequest(); deleteAll.deleteByQuery("*:*"); solrClient.request(deleteAll, collection); } } catch (SolrServerException e) { logger.error("Unable to clear storage from index due to server error on Solr.", e); throw new PermanentBackendException(e); } catch (IOException e) { logger.error("Unable to clear storage from index due to low-level I/O error.", e); throw new PermanentBackendException(e); } catch (Exception e) { logger.error("Unable to clear storage from index due to general error.", e); throw new PermanentBackendException(e); } }
private void commitDeletes(String collectionName, List<String> deleteIds) throws SolrServerException, IOException { if (deleteIds.size() == 0) return; solrClient.request(newUpdateRequest().deleteById(deleteIds), collectionName); }
protected void createAliasIfNotExist(CloudSolrClient client, String collectionName, String aliasName) { Aliases aliases = client.getZkStateReader().getAliases(); Map<String, String> aliasCollectionMap = aliases.getCollectionAliasMap(); if (!aliasCollectionMap.containsKey(aliasName)) { try { new CollectionAdminRequest.CreateAlias().setAliasName(aliasName) .setAliasedCollections(collectionName).process(client); } catch (SolrServerException e) { throw ExceptionHelper.refineException(e); } catch (IOException e) { throw ExceptionHelper.refineException(e); } } }
private static void createCollectionIfNotExists(CloudSolrClient client, Configuration config, String collection) throws IOException, SolrServerException, KeeperException, InterruptedException { if (!checkIfCollectionExists(client, collection)) { Integer numShards = config.get(NUM_SHARDS); Integer maxShardsPerNode = config.get(MAX_SHARDS_PER_NODE); Integer replicationFactor = config.get(REPLICATION_FACTOR); // Ideally this property used so a new configset is not uploaded for every single // index (collection) created in solr. // if a generic configSet is not set, make the configset name the same as the collection. // This was the default behavior before a default configSet could be specified String genericConfigSet = config.has(SOLR_DEFAULT_CONFIG) ? config.get(SOLR_DEFAULT_CONFIG):collection; CollectionAdminRequest.Create createRequest = new CollectionAdminRequest.Create(); createRequest.setConfigName(genericConfigSet); createRequest.setCollectionName(collection); createRequest.setNumShards(numShards); createRequest.setMaxShardsPerNode(maxShardsPerNode); createRequest.setReplicationFactor(replicationFactor); CollectionAdminResponse createResponse = createRequest.process(client); if (createResponse.isSuccess()) { logger.trace("Collection {} successfully created.", collection); } else { throw new SolrServerException(Joiner.on("\n").join(createResponse.getErrorMessages())); } } waitForRecoveriesToFinish(client, collection); }
@Override public SolrRequest toSolrRequest(List<? extends ITuple> tuples) throws SolrMapperException { List<SolrInputDocument> docs = new LinkedList<>(); for (ITuple tuple : tuples) { docs.add(buildDocument(tuple)); } UpdateRequest request = new UpdateRequest(); request.add(docs); log.debug("Created SolrRequest with content: {}", docs); return request; }
private void deleteIndividualFieldsFromIndex(String collectionName, String keyIdField, String docId, HashSet<IndexEntry> fieldDeletions) throws SolrServerException, IOException { if (fieldDeletions.isEmpty()) return; Map<String, String> fieldDeletes = new HashMap<String, String>(1) {{ put("set", null); }}; SolrInputDocument doc = new SolrInputDocument(); doc.addField(keyIdField, docId); StringBuilder sb = new StringBuilder(); for (IndexEntry fieldToDelete : fieldDeletions) { doc.addField(fieldToDelete.field, fieldDeletes); sb.append(fieldToDelete).append(","); } if (logger.isTraceEnabled()) logger.trace("Deleting individual fields [{}] for document {}", sb.toString(), docId); UpdateRequest singleDocument = newUpdateRequest(); singleDocument.add(doc); solrClient.request(singleDocument, collectionName); }
StopWatch timer = new StopWatch(true); final QueryRequest req = new QueryRequest(solrQuery); if (isBasicAuthEnabled()) { req.setBasicAuthCredentials(getUsername(), getPassword()); final QueryResponse response = req.process(getSolrClient()); final SolrDocumentList documentList = response.getResults();
public QueryRequest getCreateCollectionsRequest(String name, int numShards, int replicationFactor) { ModifiableSolrParams params = new ModifiableSolrParams(); params.set(SolrConstants.REQUEST_ACTION, CollectionParams.CollectionAction.CREATE.name()); params.set(SolrConstants.REQUEST_NAME, name); params.set(SolrConstants.REQUEST_NUM_SHARDS, numShards); params.set(SolrConstants.REQUEST_REPLICATION_FACTOR, replicationFactor); params.set(SolrConstants.REQUEST_COLLECTION_CONFIG_NAME, name); QueryRequest request = new QueryRequest(params); request.setPath(SolrConstants.REQUEST_COLLECTIONS_PATH); return request; }
final QueryRequest req = new QueryRequest(solrQuery); if (isBasicAuthEnabled()) { req.setBasicAuthCredentials(getUsername(), getPassword()); final QueryResponse response = req.process(getSolrClient()); timer.stop();