/** * Sets up our SolrServer. Presumes the existence of either * "warc.solr.zookepers" or "warc.solr.servers" in the config. */ @Override public void configure(JobConf job) { log.info("Configuring reducer, including Solr connection..."); // Get config from job property: Config conf = ConfigFactory.parseString(job .get(WARCIndexerRunner.CONFIG_PROPERTIES)); this.dummyRun = conf.getBoolean("warc.solr.dummy_run"); this.batchSize = conf.getInt("warc.solr.batch_size"); this.useEmbeddedServer = conf.getBoolean("warc.solr.hdfs"); if (job.get("mapred.output.oai-pmh") != null) this.exportXml = Boolean.parseBoolean(job .get("mapred.output.oai-pmh")); // Decide between to-HDFS and to-SolrCloud indexing modes: solrServer = new SolrWebServer(conf).getSolrServer(); log.info("Initialisation complete."); }
for (SolrInputDocument doc : docs) { try { solr.updateSolrDoc(doc); } catch (Exception e) { log.error( solr.add(docs);
private static void commit( SolrWebServer solrWeb) { // Commit any Solr Updates if( solrWeb != null ) { try { final long start = System.nanoTime(); solrWeb.commit(); Instrument.timeRel("WARCIndexerCommand.main#total", "WARCIndexerCommand.commit#success", start); } catch( SolrServerException s ) { log.warn( "SolrServerException when committing.", s ); } catch( IOException i ) { log.warn( "IOException when committing.", i ); } } }
solrServer = new SolrWebServer(conf);
q.addField(SolrFields.CRAWL_DATES); try { QueryResponse results = solrServer.query(q); if (results.getResults().size() > 0) { SolrDocument fr = results.getResults().get(0);
solrServer = new SolrWebServer(conf);
SolrWebServer solrWeb = new SolrWebServer(conf);
for (SolrInputDocument doc : docs) { try { solr.updateSolrDoc(doc); } catch (Exception e) { log.error( solr.add(docs);
private static void commit( SolrWebServer solrWeb) { // Commit any Solr Updates if( solrWeb != null ) { try { final long start = System.nanoTime(); solrWeb.commit(); Instrument.timeRel("WARCIndexerCommand.main#total", "WARCIndexerCommand.commit#success", start); } catch( SolrServerException s ) { log.warn( "SolrServerException when committing.", s ); } catch( IOException i ) { log.warn( "IOException when committing.", i ); } } }
SolrWebServer solrWeb = new SolrWebServer(conf);