/** Adds an entry to document index for empty document @param docid, only if IndexEmptyDocuments is set to true. */ protected void indexEmpty(Map<String,String> docProperties) throws IOException { if (! IndexEmptyDocuments) return; /* add doc to documentindex, even though it's empty */ logger.warn("Adding empty document "+docProperties.get("docno")); docIndexBuilder.addEntryToBuffer(emptyDocIndexEntry); metaBuilder.writeDocumentEntry(docProperties); }
metaBuilder.close(); throw new Error("Meta fields in source indices must match"); if (srcFieldCount1 != srcFieldCount2) metaBuilder.close(); throw new Error("FieldCounts in source indices must match"); DocumentIndexEntry dieNew = (fieldCount > 0) ? die : new SimpleDocumentIndexEntry(die); docidOutput.addEntryToBuffer(dieNew); metaBuilder.writeDocumentEntry(metaInput1.next()); DocumentIndexEntry dieNew = (fieldCount > 0) ? die : new SimpleDocumentIndexEntry(die); docidOutput.addEntryToBuffer(dieNew); metaBuilder.writeDocumentEntry(metaInput2.next()); metaBuilder.close(); IndexUtil.close(docidInput1); IndexUtil.close(docidInput2);
metaBuilder.close(); currentIndex.flush();
metaBuilder.close(); throw new Error("Meta fields in source indices must match"); if (srcFieldCount1 != srcFieldCount2) metaBuilder.close(); throw new Error("FieldCounts in source indices must match"); destIndex.getPrefix() + ".direct" + compressionDirectConfig.getStructureFileExtension()); } catch (Exception e) { metaBuilder.close(); throw new Error("Couldn't create specified DirectInvertedOutputStream", e); metaBuilder.writeDocumentEntry(metaInput1.getAllItems(sourceDocid)); sourceDocid++; metaBuilder.writeDocumentEntry(metaInput2.getAllItems(sourceDocid)); sourceDocid++; metaInput2.close(); metaBuilder.close(); dfOutput.close(); docidOutput.finishedCollections();
metaBuilder.close(); currentIndex.flush();
b.writeDocumentEntry(dataOne); b.close(); b = null; finishedCreatingMeta(index, name);
/** Adds an entry to document index for empty document @param docid, only if IndexEmptyDocuments is set to true. */ protected void indexEmpty(Map<String,String> docProperties) throws IOException { if (seenDocnos.contains(docProperties.get("docno"))) return; else seenDocnos.add(docProperties.get("docno")); if (! IndexEmptyDocuments) return; /* add doc to documentindex, even though it's empty */ logger.warn("Adding empty document "+docProperties.get("docno")); docIndexBuilder.addEntryToBuffer(emptyDocIndexEntry); metaBuilder.writeDocumentEntry(docProperties); }
metaBuilder.close(); } catch (IOException ioe) { logger.error("Could not finish MetaIndexBuilder: ", ioe);
/** * {@inheritDoc}. * This implementation only places content in the runs in memory, which will eventually be flushed to disk. */ @Override protected void indexDocument(Map<String,String> docProperties, DocumentPostingList termsInDocument) throws Exception { if (seenDocnos.contains(docProperties.get("docno"))) return; else seenDocnos.add(docProperties.get("docno")); if (termsInDocument.getDocumentLength() > 0) { numberOfDocsSinceCheck++; numberOfDocsSinceFlush++; checkFlush(); mp.addTerms(termsInDocument, currentId); DocumentIndexEntry die = termsInDocument.getDocumentStatistics(); docIndexBuilder.addEntryToBuffer((FieldScore.FIELDS_COUNT > 0) ? die : new SimpleDocumentIndexEntry(die)); metaBuilder.writeDocumentEntry(docProperties); currentId++; numberOfDocuments++; } }
metaBuilder.close(); } catch (IOException ioe) { logger.error("Could not finish MetaIndexBuilder: ", ioe);
/** * {@inheritDoc}. * This implementation only places content in the runs in memory, which will eventually be flushed to disk. */ @Override protected void indexDocument(Map<String,String> docProperties, DocumentPostingList termsInDocument) throws Exception { if (termsInDocument.getDocumentLength() > 0) { numberOfDocsSinceCheck++; numberOfDocsSinceFlush++; checkFlush(); mp.addTerms(termsInDocument, currentId); DocumentIndexEntry die = termsInDocument.getDocumentStatistics(); docIndexBuilder.addEntryToBuffer((FieldScore.FIELDS_COUNT > 0) ? die : new SimpleDocumentIndexEntry(die)); metaBuilder.writeDocumentEntry(docProperties); currentId++; numberOfDocuments++; } }
/** * This adds a document to the direct and document indexes, as well * as it's terms to the lexicon. Handled internally by the methods * indexFieldDocument and indexNoFieldDocument. * @param docProperties Map<String,String> properties of the document * @param _termsInDocument DocumentPostingList the terms in the document. * */ protected void indexDocument(Map<String,String> docProperties, DocumentPostingList _termsInDocument) throws Exception { /* add words to lexicontree */ lexiconBuilder.addDocumentTerms(_termsInDocument); /* add doc postings to the direct index */ BitIndexPointer dirIndexPost = directIndexBuilder.writePostings(_termsInDocument.getPostings2(termCodes)); /* add doc to documentindex */ DocumentIndexEntry die = _termsInDocument.getDocumentStatistics(); die.setBitIndexPointer(dirIndexPost); docIndexBuilder.addEntryToBuffer(die); /** add doc metadata to index */ metaBuilder.writeDocumentEntry(docProperties); }
/** * This adds a document to the direct and document indexes, as well * as it's terms to the lexicon. Handled internally by the methods * indexFieldDocument and indexNoFieldDocument. * @param docProperties Map<String,String> properties of the document * @param _termsInDocument DocumentPostingList the terms in the document. * */ protected void indexDocument(Map<String,String> docProperties, DocumentPostingList _termsInDocument) throws Exception { /* add words to lexicontree */ lexiconBuilder.addDocumentTerms(_termsInDocument); /* add doc postings to the direct index */ BitIndexPointer dirIndexPost = directIndexBuilder.writePostings(_termsInDocument.getPostings2(termCodes)); //.addDocument(termsInDocument.getPostings()); /* add doc to documentindex */ DocumentIndexEntry die = _termsInDocument.getDocumentStatistics(); die.setBitIndexPointer(dirIndexPost); docIndexBuilder.addEntryToBuffer(die); /** add doc metadata to index */ metaBuilder.writeDocumentEntry(docProperties); }