public void listAllDocuments() throws IOException { setIndexReaderSearcher(); for(int i = 0; i < reader.numDocs(); i++){ Document d = searcher.doc(i); // byte[] sent = d.getBinaryValue("tokens").bytes; // List<CoreLabel> tokens = readProtoBufAnnotation(sent); System.out.println(d.get("sentid")); } }
public static LuceneSentenceIndex loadIndex(Properties props, Set<String> stopwords, String dir, Function<CoreLabel, Map<String, String>> transformSentenceToString) { try { LuceneSentenceIndex sentindex = new LuceneSentenceIndex(props, stopwords, dir, transformSentenceToString); sentindex.setIndexReaderSearcher(); System.out.println("Number of documents read from the index " + dir + " are " + sentindex.reader.numDocs()); sentindex.numAllSentences += sentindex.reader.numDocs(); return sentindex; } catch (IOException e) { throw new RuntimeException(e); } }
public static LuceneSentenceIndex createIndex(Map<String, List<CoreLabel>> sentences, Properties props, Set<String> stopWords, String indexDiskDir, Function<CoreLabel, Map<String, String>> transformer) { try{ LuceneSentenceIndex sentindex = new LuceneSentenceIndex(props, stopWords, indexDiskDir, transformer); System.out.println("Creating lucene index at " + indexDiskDir); IOUtils.deleteDirRecursively(sentindex.indexDir); if(sentences!= null){ sentindex.setIndexWriter(); sentindex.add(sentences, true); sentindex.closeIndexWriter(); sentindex.setIndexReaderSearcher(); System.out.println("Number of documents added are " + sentindex.reader.numDocs()); sentindex.numAllSentences += sentindex.reader.numDocs(); } return sentindex; }catch(IOException e){ throw new RuntimeException(e); } }
result.put("current", reader.isCurrent()); result.put("disk_size", Utils.directorySize(reader.directory())); result.put("doc_count", reader.numDocs()); result.put("doc_del_count", reader.numDeletedDocs()); result.put("uuid", state.getUuid());
@Override public boolean hasNext() throws IOException, CollectionException { return (docNum < ireader.numDocs()) && (maxWords < 0 || wordNum < maxWords); }
@Override public boolean hasNext() throws IOException, CollectionException { return (docNum < ireader.numDocs()) && (maxWords < 0 || wordNum < maxWords); }
public Map<String, Map<String, String>> getSearchIndexInfo() { Map<String, Map<String, String>> result = new LinkedHashMap<>(); for (Map.Entry<String, DirectoryReader> e : readerList.entrySet()) { String dbname = e.getKey(); DirectoryReader diskReader = e.getValue(); Map<String, String> indexInfo = new LinkedHashMap<>(); indexInfo.put("count", diskReader.numDocs() + ""); result.put(dbname, indexInfo); } return result; }
@Override public int getSize() { ensureOpen(); return indexReader.numDocs(); }
@Override public int getSize() { ensureOpen(); return indexReader.numDocs(); }
public int getNumberOfDocuments() { try { int numDocs = dirReader.getDocCount(fieldName); if (numDocs < 0) { LOGGER.warn("Coudln't get the number of documents with the indexed field \"" + fieldName + "\". Using the number of all documents in the index instead."); } } catch (IOException e) { LOGGER.warn("Coudln't get the number of documents with the indexed field \"" + fieldName + "\". Using the number of all documents in the index instead.", e); } return dirReader.numDocs(); }
/** * Counts the documents * @param directory Directory * @return the number of docs,including all segments * @throws IOException */ public static int numDocs(Directory directory) throws IOException { try (DirectoryReader reader = DirectoryReader.open(directory)) { return reader.numDocs(); } }
@Override public Progress[] getProgress() { return new Progress[]{ (maxWords < 0 ? new ProgressImpl(docNum, ireader.numDocs(), "Documents") : new ProgressImpl(wordNum, maxWords, "Words"))}; }
@Override public Progress[] getProgress() { return new Progress[]{ (maxWords < 0 ? new ProgressImpl(docNum, ireader.numDocs(), "Documents") : new ProgressImpl(wordNum, maxWords, "Words"))}; }
public int getLiveRecords() { checkIfChanged(); try(Searcher searcher = acquireSearcher()) { DirectoryReader reader = (DirectoryReader) searcher.searcher.getIndexReader(); return reader.numDocs(); } }
void printIndexStats() throws IOException { Fields fields = MultiFields.getFields(reader); Terms terms = fields.terms(LuceneDocumentGenerator.FIELD_BODY); System.out.println("Index statistics"); System.out.println("----------------"); System.out.println("documents: " + reader.numDocs()); System.out.println("documents (non-empty): " + reader.getDocCount(LuceneDocumentGenerator.FIELD_BODY)); System.out.println("unique terms: " + terms.size()); System.out.println("total terms: " + reader.getSumTotalTermFreq(LuceneDocumentGenerator.FIELD_BODY)); System.out.println("stored fields:"); FieldInfos fieldInfos = MultiFields.getMergedFieldInfos(reader); for (String fd : fields) { FieldInfo fi = fieldInfos.fieldInfo(fd); System.out.println(" " + fd + " (" + "indexOption: " + fi.getIndexOptions() + ", hasVectors: " + fi.hasVectors() + ")"); } }
dirStatus.numDocs = dirReader.numDocs(); log.debug("[{}][{}] DirectoryReader can be opened", indexPath, dirName); closer.register(dirReader);
public SegmentCountResponse getNumberOfDocs() throws IOException { openReaderIfChanges(); int count = directoryReader.numDocs(); return SegmentCountResponse.newBuilder().setNumberOfDocs(count).setSegmentNumber(segmentNumber).build(); }
dirStatus.numDocs = dirReader.numDocs(); log.debug("[{}][{}] DirectoryReader can be opened", indexPath, dirName); closer.register(dirReader);
DirectoryReader r = DirectoryReader.open(taxoDir); try { final int size = r.numDocs(); final OrdinalMap ordinalMap = map; ordinalMap.setSize(size);
DirectoryReader r = DirectoryReader.open(taxoDir); try { final int size = r.numDocs(); final OrdinalMap ordinalMap = map; ordinalMap.setSize(size);