Refine search
@Override public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException { final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION); try (IndexOutput output = directory.createOutput(fileName, context)) { CodecUtil.writeIndexHeader(output, Lucene50FieldInfosFormat.CODEC_NAME, Lucene50FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix); output.writeVInt(infos.size()); for (FieldInfo fi : infos) { fi.checkConsistency(); output.writeString(fi.name); output.writeVInt(fi.number); byte bits = 0x0; if (fi.hasVectors()) bits |= STORE_TERMVECTOR; if (fi.omitsNorms()) bits |= OMIT_NORMS; if (fi.hasPayloads()) bits |= STORE_PAYLOADS; output.writeByte(bits); output.writeByte(indexOptionsByte(fi.getIndexOptions())); // pack the DV type and hasNorms in one byte output.writeByte(docValuesByte(fi.getDocValuesType())); output.writeLong(fi.getDocValuesGen()); output.writeMapOfStrings(fi.attributes()); } CodecUtil.writeFooter(output); } }
fi = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValues, dvGen, new HashMap<>(), dataDimensionCount, indexDimensionCount, dimensionNumBytes, isSoftDeletesField); assert !byName.containsKey(fi.name); globalFieldNumbers.verifyConsistent(Integer.valueOf(fi.number), fi.name, fi.getDocValuesType()); byName.put(fi.name, fi); } else { fi.update(storeTermVector, omitNorms, storePayloads, indexOptions, dataDimensionCount, indexDimensionCount, dimensionNumBytes); boolean updateGlobal = fi.getDocValuesType() == DocValuesType.NONE; if (updateGlobal) { fi.setDocValuesType(docValues); // this will also perform the consistency check. fi.setDocValuesGen(dvGen);
int docValuesCount = 0; int normsCount = 0; for (FieldInfo info : reader.getFieldInfos()) { if (info.getIndexOptions() != IndexOptions.NONE) { reader.terms(info.name); indexedCount++; if (info.hasNorms()) { reader.getNormValues(info.name); normsCount++; if (info.getDocValuesType() != DocValuesType.NONE) { switch(info.getDocValuesType()) { case NUMERIC: reader.getNumericDocValues(info.name);
public FieldInfo add(FieldInfo fi, long dvGen) { // IMPORTANT - reuse the field number if possible for consistent field numbers across segments return addOrUpdateInternal(fi.name, fi.number, fi.hasVectors(), fi.omitsNorms(), fi.hasPayloads(), fi.getIndexOptions(), fi.getDocValuesType(), dvGen, fi.getPointDataDimensionCount(), fi.getPointIndexDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField()); }
public boolean canReuse(IndexInput docIn, FieldInfo fieldInfo) { return docIn == startDocIn && indexHasOffsets == (fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0) && indexHasPayloads == fieldInfo.hasPayloads(); }
/** * Loads or returns the already loaded the global field number map for this {@link SegmentInfos}. * If this {@link SegmentInfos} has no global field number map the returned instance is empty */ private FieldNumbers getFieldNumberMap() throws IOException { final FieldNumbers map = new FieldNumbers(config.softDeletesField); for(SegmentCommitInfo info : segmentInfos) { FieldInfos fis = readFieldInfos(info); for(FieldInfo fi : fis) { map.addOrGet(fi.name, fi.number, fi.getDocValuesType(), fi.getPointDataDimensionCount(), fi.getPointIndexDimensionCount(), fi.getPointNumBytes(), fi.isSoftDeletesField()); } } return map; }
try (IndexOutput output = directory.createOutput(fileName, context)) { CodecUtil.writeIndexHeader(output, Lucene60FieldInfosFormat.CODEC_NAME, Lucene60FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix); output.writeVInt(infos.size()); for (FieldInfo fi : infos) { fi.checkConsistency(); if (fi.hasVectors()) bits |= STORE_TERMVECTOR; if (fi.omitsNorms()) bits |= OMIT_NORMS; if (fi.hasPayloads()) bits |= STORE_PAYLOADS; if (fi.isSoftDeletesField()) bits |= SOFT_DELETES_FIELD; output.writeByte(bits); output.writeByte(indexOptionsByte(fi.getIndexOptions())); output.writeByte(docValuesByte(fi.getDocValuesType())); output.writeLong(fi.getDocValuesGen()); output.writeMapOfStrings(fi.attributes()); output.writeVInt(fi.getPointDataDimensionCount()); if (fi.getPointDataDimensionCount() != 0) { output.writeVInt(fi.getPointIndexDimensionCount()); output.writeVInt(fi.getPointNumBytes());
this.mapping = mapping; ArrayList<FieldInfo> filteredInfos = new ArrayList<>(); for (FieldInfo fi : in.getFieldInfos()) { DocValuesType type = fi.getDocValuesType(); if (fi.getIndexOptions() != IndexOptions.NONE && fi.getDocValuesType() == DocValuesType.NONE) { Type t = mapping.get(fi.name); if (t != null) { filteredInfos.add(new FieldInfo(fi.name, fi.number, fi.hasVectors(), fi.omitsNorms(), fi.hasPayloads(), fi.getIndexOptions(), type, fi.getDocValuesGen(), fi.attributes())); fieldInfos = new FieldInfos(filteredInfos.toArray(new FieldInfo[filteredInfos.size()]));
FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field); final DocIdSetIterator iterator; if (fieldInfo != null) { switch (fieldInfo.getDocValuesType()) { case NONE: iterator = null; break; case NUMERIC: iterator = reader.getNumericDocValues(field); break; case BINARY: iterator = reader.getBinaryDocValues(field); break; case SORTED:
/** * @return the {@code NumericDocValues} for a given field * @throws IllegalArgumentException if this field is not indexed with numeric doc values */ public NumericDocValues readDocValues( String field ) { try { NumericDocValues dv = context.reader().getNumericDocValues( field ); if ( dv == null ) { FieldInfo fi = context.reader().getFieldInfos().fieldInfo( field ); DocValuesType actual = null; if ( fi != null ) { actual = fi.getDocValuesType(); } throw new IllegalStateException( "The field '" + field + "' is not indexed properly, expected NumericDV, but got '" + actual + "'" ); } return dv; } catch ( IOException e ) { throw new RuntimeException( e ); } } }
private FieldInfoStatus(final FieldInfo info) { this(info.number, info.omitsNorms(), info.hasNorms(), info.hasPayloads(), info.hasVectors(), info.getDocValuesGen(), info.getDocValuesType(), info.getIndexOptions(), info.getPointDimensionCount(), info.getPointNumBytes()); }
if (parallelReaders.length > 0) { final LeafReader first = parallelReaders[0]; this.maxDoc = first.maxDoc(); this.numDocs = first.numDocs(); this.hasDeletions = first.hasDeletions(); } else { this.maxDoc = this.numDocs = 0; .map(r -> r.getFieldInfos().getSoftDeletesField()) .filter(Objects::nonNull).findAny().orElse(null); builder.add(fieldInfo, fieldInfo.getDocValuesGen()); fieldToReader.put(fieldInfo.name, reader); if (fieldInfo.hasVectors()) { tvFieldToReader.put(fieldInfo.name, reader); if (fieldInfo.getIndexOptions() != IndexOptions.NONE) { termsFieldToReader.put(fieldInfo.name, reader);
private FieldInfo getDVField(String field, DocValuesType type) { FieldInfo fi = getFieldInfos().fieldInfo(field); if (fi == null) { // Field does not exist return null; } if (fi.getDocValuesType() == DocValuesType.NONE) { // Field was not indexed with doc values return null; } if (fi.getDocValuesType() != type) { // Field DocValues are different than requested type return null; } return fi; }
if (this.filteredNames.contains(fi.name)) { this.filtered.add(fi); hasVectors |= fi.hasVectors(); hasProx |= fi.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0; hasFreq |= fi.getIndexOptions() != IndexOptions.DOCS; hasOffsets |= fi.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0; hasNorms |= fi.hasNorms(); hasDocValues |= fi.getDocValuesType() != DocValuesType.NONE; hasPayloads |= fi.hasPayloads(); hasPointValues |= (fi.getPointDataDimensionCount() != 0);
private static void checkField(LeafReader in, String field, DocValuesType... expected) { FieldInfo fi = in.getFieldInfos().fieldInfo(field); if (fi != null) { DocValuesType actual = fi.getDocValuesType(); throw new IllegalStateException("unexpected docvalues type " + actual + " for field '" + field + "' " + (expected.length == 1 ? "(expected=" + expected[0] : "(expected one of " + Arrays.toString(expected)) + "). " + "Re-index with correct docvalues type."); } }
private void storeDocValues(Info info, DocValuesType docValuesType, Object docValuesValue) { String fieldName = info.fieldInfo.name; DocValuesType existingDocValuesType = info.fieldInfo.getDocValuesType(); if (existingDocValuesType == DocValuesType.NONE) { info.fieldInfo = new FieldInfo( info.fieldInfo.name, info.fieldInfo.number, info.fieldInfo.hasVectors(), info.fieldInfo.hasPayloads(), info.fieldInfo.hasPayloads(), info.fieldInfo.getIndexOptions(), docValuesType, -1, info.fieldInfo.attributes(), info.fieldInfo.getPointDataDimensionCount(), info.fieldInfo.getPointIndexDimensionCount(), info.fieldInfo.getPointNumBytes(), info.fieldInfo.isSoftDeletesField() ); } else if (existingDocValuesType != docValuesType) {
@Override public Scorer scorer(LeafReaderContext context) throws IOException { FieldInfos fieldInfos = context.reader().getFieldInfos(); FieldInfo fieldInfo = fieldInfos.fieldInfo(field); if (fieldInfo == null || fieldInfo.hasNorms() == false) { return null; } LeafReader reader = context.reader(); DocIdSetIterator iterator = reader.getNormValues(field); return new ConstantScoreScorer(this, score(), iterator); }
return null; } else if (size == 1) { return leaves.get(0).reader().getBinaryDocValues(field); FieldInfo fieldInfo = leaf.reader().getFieldInfos().fieldInfo(field); if (fieldInfo != null) { DocValuesType dvType = fieldInfo.getDocValuesType(); if (dvType == DocValuesType.BINARY) { anyReal = true;
LeafReaderContext context = in.leaves().get(i); final LeafReader reader = context.reader(); final FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field); if (fieldInfo != null && fieldInfo.getDocValuesType() != DocValuesType.SORTED) { return null; SortedDocValues v = reader.getSortedDocValues(field); if (v == null) { v = DocValues.emptySorted();
/** * Returns {@code true} if the specified docvalues fields have not been updated */ public static boolean isCacheable(LeafReaderContext ctx, String... fields) { for (String field : fields) { FieldInfo fi = ctx.reader().getFieldInfos().fieldInfo(field); if (fi != null && fi.getDocValuesGen() > -1) return false; } return true; } }