/** * Translates any added {@link FacetField}s into normal fields for indexing; * only use this version if you did not add any taxonomy-based fields ( * {@link FacetField} or {@link AssociationFacetField}). * * <p> * <b>NOTE:</b> you should add the returned document to IndexWriter, not the * input one! */ public Document build(Document doc) throws IOException { return build(null, doc); }
/** * Translates any added {@link FacetField}s into normal fields for indexing; * only use this version if you did not add any taxonomy-based fields ( * {@link FacetField} or {@link AssociationFacetField}). * * <p> * <b>NOTE:</b> you should add the returned document to IndexWriter, not the * input one! */ public Document build(Document doc) throws IOException { return build(null, doc); }
public Document buildDocument(Document doc) throws IOException { return config.build(taxonomyWriter, doc); }
public Document build(Document doc) { if ( facetConfig == null ) { return doc; } else { try { return facetConfig.build( doc ); } catch (IOException e) { throw log.errorDuringFacetingIndexing( e ); } } }
private Document getFacetedDoc() throws IOException { final FacetsConfig facetsConfig = fieldMap.getFacetsConfig(documentBuilder.dimensions); return facetsConfig.build(taxonomyWriter, documentBuilder.document); }
public Document build(Document doc) { if ( facetConfig == null ) { return doc; } else { try { return facetConfig.build( doc ); } catch (IOException e) { throw log.errorDuringFacetingIndexing( e ); } } }
/** Build the example index. */ private void index() throws IOException { IndexWriter indexWriter = new IndexWriter(indexDir, new IndexWriterConfig( new WhitespaceAnalyzer()).setOpenMode(OpenMode.CREATE)); Document doc = new Document(); doc.add(new SortedSetDocValuesFacetField("Author", "Bob")); doc.add(new SortedSetDocValuesFacetField("Publish Year", "2010")); indexWriter.addDocument(config.build(doc)); doc = new Document(); doc.add(new SortedSetDocValuesFacetField("Author", "Lisa")); doc.add(new SortedSetDocValuesFacetField("Publish Year", "2010")); indexWriter.addDocument(config.build(doc)); doc = new Document(); doc.add(new SortedSetDocValuesFacetField("Author", "Lisa")); doc.add(new SortedSetDocValuesFacetField("Publish Year", "2012")); indexWriter.addDocument(config.build(doc)); doc = new Document(); doc.add(new SortedSetDocValuesFacetField("Author", "Susan")); doc.add(new SortedSetDocValuesFacetField("Publish Year", "2012")); indexWriter.addDocument(config.build(doc)); doc = new Document(); doc.add(new SortedSetDocValuesFacetField("Author", "Frank")); doc.add(new SortedSetDocValuesFacetField("Publish Year", "1999")); indexWriter.addDocument(config.build(doc)); indexWriter.close(); }
/** Build the example index. */ private void index() throws IOException { IndexWriter indexWriter = new IndexWriter(indexDir, new IndexWriterConfig( new WhitespaceAnalyzer()).setOpenMode(OpenMode.CREATE)); // Writes facet ords to a separate directory from the main index DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir); Document doc = new Document(); doc.add(new FacetField("Author", "Bob")); doc.add(new FacetField("Publish Date", "2010", "10", "15")); indexWriter.addDocument(config.build(taxoWriter, doc)); doc = new Document(); doc.add(new FacetField("Author", "Lisa")); doc.add(new FacetField("Publish Date", "2010", "10", "20")); indexWriter.addDocument(config.build(taxoWriter, doc)); doc = new Document(); doc.add(new FacetField("Author", "Lisa")); doc.add(new FacetField("Publish Date", "2012", "1", "1")); indexWriter.addDocument(config.build(taxoWriter, doc)); doc = new Document(); doc.add(new FacetField("Author", "Susan")); doc.add(new FacetField("Publish Date", "2012", "1", "7")); indexWriter.addDocument(config.build(taxoWriter, doc)); doc = new Document(); doc.add(new FacetField("Author", "Frank")); doc.add(new FacetField("Publish Date", "1999", "5", "5")); indexWriter.addDocument(config.build(taxoWriter, doc)); indexWriter.close(); taxoWriter.close(); }
/** Build the example index. */ private void index() throws IOException { IndexWriter indexWriter = new IndexWriter(indexDir, new IndexWriterConfig( new WhitespaceAnalyzer()).setOpenMode(OpenMode.CREATE)); // Writes facet ords to a separate directory from the main index DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir); Document doc = new Document(); doc.add(new TextField("c", "foo bar", Store.NO)); doc.add(new NumericDocValuesField("popularity", 5L)); doc.add(new FacetField("A", "B")); indexWriter.addDocument(config.build(taxoWriter, doc)); doc = new Document(); doc.add(new TextField("c", "foo foo bar", Store.NO)); doc.add(new NumericDocValuesField("popularity", 3L)); doc.add(new FacetField("A", "C")); indexWriter.addDocument(config.build(taxoWriter, doc)); indexWriter.close(); taxoWriter.close(); }
/** Build the example index. */ private void index() throws IOException { IndexWriter indexWriter = new IndexWriter(indexDir, new IndexWriterConfig( new WhitespaceAnalyzer()).setOpenMode(OpenMode.CREATE)); // Writes facet ords to a separate directory from the main index DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir); Document doc = new Document(); doc.add(new FacetField("Author", "Bob")); doc.add(new FacetField("Publish Date", "2010", "10", "15")); indexWriter.addDocument(config.build(taxoWriter, doc)); doc = new Document(); doc.add(new FacetField("Author", "Lisa")); doc.add(new FacetField("Publish Date", "2010", "10", "20")); indexWriter.addDocument(config.build(taxoWriter, doc)); doc = new Document(); doc.add(new FacetField("Author", "Lisa")); doc.add(new FacetField("Publish Date", "2012", "1", "1")); indexWriter.addDocument(config.build(taxoWriter, doc)); doc = new Document(); doc.add(new FacetField("Author", "Susan")); doc.add(new FacetField("Publish Date", "2012", "1", "7")); indexWriter.addDocument(config.build(taxoWriter, doc)); doc = new Document(); doc.add(new FacetField("Author", "Frank")); doc.add(new FacetField("Publish Date", "1999", "5", "5")); indexWriter.addDocument(config.build(taxoWriter, doc)); indexWriter.close(); taxoWriter.close(); }
/** Build the example index. */ private void index() throws IOException { IndexWriterConfig iwc = new IndexWriterConfig(new WhitespaceAnalyzer()).setOpenMode(OpenMode.CREATE); IndexWriter indexWriter = new IndexWriter(indexDir, iwc); // Writes facet ords to a separate directory from the main index DirectoryTaxonomyWriter taxoWriter = new DirectoryTaxonomyWriter(taxoDir); Document doc = new Document(); // 3 occurrences for tag 'lucene' doc.add(new IntAssociationFacetField(3, "tags", "lucene")); // 87% confidence level of genre 'computing' doc.add(new FloatAssociationFacetField(0.87f, "genre", "computing")); indexWriter.addDocument(config.build(taxoWriter, doc)); doc = new Document(); // 1 occurrence for tag 'lucene' doc.add(new IntAssociationFacetField(1, "tags", "lucene")); // 2 occurrence for tag 'solr' doc.add(new IntAssociationFacetField(2, "tags", "solr")); // 75% confidence level of genre 'computing' doc.add(new FloatAssociationFacetField(0.75f, "genre", "computing")); // 34% confidence level of genre 'software' doc.add(new FloatAssociationFacetField(0.34f, "genre", "software")); indexWriter.addDocument(config.build(taxoWriter, doc)); indexWriter.close(); taxoWriter.close(); }
@Override public int doLogic() throws Exception { if (config != null) { List<FacetField> facets = new ArrayList<>(); getRunData().getFacetSource().getNextFacets(facets); for(FacetField ff : facets) { doc.add(ff); } doc = config.build(getRunData().getTaxonomyWriter(), doc); } return super.doLogic(); } }
private Document generateIndexDoc(Record record, Map<String, ColumnDefinition> columns, TaxonomyWriter taxonomyWriter) throws AnalyticsIndexException, IOException { Document doc = new Document(); FacetsConfig config = new FacetsConfig(); FieldType numericFieldType = getLuceneNumericFieldType(FieldType.NumericType.LONG); doc.add(new StringField(INDEX_ID_INTERNAL_FIELD, record.getId(), Store.YES)); doc.add(new LongField(INDEX_INTERNAL_TIMESTAMP_FIELD, record.getTimestamp(), numericFieldType)); /* make the best effort to store in the given timestamp, or else, * fall back to a compatible format, or else, lastly, string */ String name; for (Map.Entry<String, ColumnDefinition> entry : columns.entrySet()) { name = entry.getKey(); this.checkAndAddDocEntry(doc, entry.getValue().getType(), name, record.getValue(name)); if (entry.getValue().isFacet()) { this.checkAndAddTaxonomyDocEntries(doc, name, record, config); } } return config.build(taxonomyWriter, doc); }
doc.add(new SortedSetDocValuesFacetField("Author", "Bob")); doc.add(new SortedSetDocValuesFacetField("Publish Year", "2010")); indexWriter.addDocument(config.build(doc)); indexWriter.addDocument(config.build(doc)); indexWriter.addDocument(config.build(doc)); indexWriter.addDocument(config.build(doc)); indexWriter.addDocument(config.build(doc));
public void index(String uniqueId, long timestamp, org.bson.Document mongoDocument, List<Metadata> metadataList) throws Exception { reopenIndexWritersIfNecessary(); Document luceneDocument = new Document(); addStoredFieldsForDocument(mongoDocument, luceneDocument); luceneDocument.add(new StringField(LumongoConstants.ID_FIELD, uniqueId, Store.YES)); luceneDocument.add(new LongPoint(LumongoConstants.TIMESTAMP_FIELD, timestamp)); luceneDocument.add(new StoredField(LumongoConstants.TIMESTAMP_FIELD, timestamp)); if (indexConfig.getIndexSettings().getStoreDocumentInIndex()) { luceneDocument.add(new StoredField(LumongoConstants.STORED_DOC_FIELD, new BytesRef(LumongoUtil.mongoDocumentToByteArray(mongoDocument)))); org.bson.Document metadataMongoDoc = new org.bson.Document(); for (Metadata metadata : metadataList) { metadataMongoDoc.put(metadata.getKey(), metadata.getValue()); } luceneDocument.add(new StoredField(LumongoConstants.STORED_META_FIELD, new BytesRef(LumongoUtil.mongoDocumentToByteArray(metadataMongoDoc)))); } luceneDocument = facetsConfig.build(taxoWriter, luceneDocument); Term term = new Term(LumongoConstants.ID_FIELD, uniqueId); indexWriter.updateDocument(term, luceneDocument); possibleCommit(); }
@Override protected Document finalizeDoc(Document doc, boolean dirty, boolean facet) throws IOException { if (facet && isFacetingEnabled()) { doc = getFacetsConfig().build(doc); } List<IndexableField> fields = doc.getFields(); // because of LUCENE-5833 we have to merge the suggest fields into a single one Field suggestField = null; for (IndexableField f : fields) { if (FieldNames.SUGGEST.equals(f.name())) { if (suggestField == null) { suggestField = FieldFactory.newSuggestField(f.stringValue()); } else { suggestField = FieldFactory.newSuggestField(suggestField.stringValue(), f.stringValue()); } } } doc.removeFields(FieldNames.SUGGEST); if (suggestField != null) { doc.add(suggestField); } return doc; }
@Override protected Document finalizeDoc(Document doc, boolean dirty, boolean facet) throws IOException { if (facet && isFacetingEnabled()) { doc = getFacetsConfig().build(doc); } List<IndexableField> fields = doc.getFields(); // because of LUCENE-5833 we have to merge the suggest fields into a single one Field suggestField = null; for (IndexableField f : fields) { if (FieldNames.SUGGEST.equals(f.name())) { if (suggestField == null) { suggestField = FieldFactory.newSuggestField(f.stringValue()); } else { suggestField = FieldFactory.newSuggestField(suggestField.stringValue(), f.stringValue()); } } } doc.removeFields(FieldNames.SUGGEST); if (suggestField != null) { doc.add(suggestField); } return doc; }