Analyzer analyzer = fieldType.indexAnalyzer(); for (Object fieldValue : textsToHighlight) { String fieldText = fieldValue.toString();
public DocumentFieldMappers(Collection<FieldMapper> mappers, Collection<FieldAliasMapper> aliasMappers, Analyzer defaultIndex, Analyzer defaultSearch, Analyzer defaultSearchQuote) { Map<String, Mapper> fieldMappers = new HashMap<>(); Map<String, Analyzer> indexAnalyzers = new HashMap<>(); Map<String, Analyzer> searchAnalyzers = new HashMap<>(); Map<String, Analyzer> searchQuoteAnalyzers = new HashMap<>(); for (FieldMapper mapper : mappers) { fieldMappers.put(mapper.name(), mapper); MappedFieldType fieldType = mapper.fieldType(); put(indexAnalyzers, fieldType.name(), fieldType.indexAnalyzer(), defaultIndex); put(searchAnalyzers, fieldType.name(), fieldType.searchAnalyzer(), defaultSearch); put(searchQuoteAnalyzers, fieldType.name(), fieldType.searchQuoteAnalyzer(), defaultSearchQuote); } for (FieldAliasMapper aliasMapper : aliasMappers) { fieldMappers.put(aliasMapper.name(), aliasMapper); } this.fieldMappers = Collections.unmodifiableMap(fieldMappers); this.indexAnalyzer = new FieldNameAnalyzer(indexAnalyzers); this.searchAnalyzer = new FieldNameAnalyzer(searchAnalyzers); this.searchQuoteAnalyzer = new FieldNameAnalyzer(searchQuoteAnalyzers); }
protected final void doXContentAnalyzers(XContentBuilder builder, boolean includeDefaults) throws IOException { if (fieldType.tokenized() == false) { return; } if (fieldType().indexAnalyzer() == null) { if (includeDefaults) { builder.field("analyzer", "default"); } } else { boolean hasDefaultIndexAnalyzer = fieldType().indexAnalyzer().name().equals("default"); final String searchAnalyzerName = fieldType().searchAnalyzer().name(); boolean hasDifferentSearchAnalyzer = searchAnalyzerName.equals(fieldType().indexAnalyzer().name()) == false; boolean hasDifferentSearchQuoteAnalyzer = searchAnalyzerName.equals(fieldType().searchQuoteAnalyzer().name()) == false; if (includeDefaults || hasDefaultIndexAnalyzer == false || hasDifferentSearchAnalyzer || hasDifferentSearchQuoteAnalyzer) { builder.field("analyzer", fieldType().indexAnalyzer().name()); if (includeDefaults || hasDifferentSearchAnalyzer || hasDifferentSearchQuoteAnalyzer) { builder.field("search_analyzer", searchAnalyzerName); if (includeDefaults || hasDifferentSearchQuoteAnalyzer) { builder.field("search_quote_analyzer", fieldType().searchQuoteAnalyzer().name()); } } } } }
private static Analyzer getAnalyzerAtField(IndexShard indexShard, String field, @Nullable Map<String, String> perFieldAnalyzer) { MapperService mapperService = indexShard.mapperService(); Analyzer analyzer; if (perFieldAnalyzer != null && perFieldAnalyzer.containsKey(field)) { analyzer = mapperService.getIndexAnalyzers().get(perFieldAnalyzer.get(field).toString()); } else { MappedFieldType fieldType = mapperService.fullName(field); if (fieldType instanceof KeywordFieldMapper.KeywordFieldType) { KeywordFieldMapper.KeywordFieldType keywordFieldType = (KeywordFieldMapper.KeywordFieldType) fieldType; analyzer = keywordFieldType.normalizer() == null ? keywordFieldType.indexAnalyzer() : keywordFieldType.normalizer(); } else { analyzer = fieldType.indexAnalyzer(); } } if (analyzer == null) { analyzer = mapperService.getIndexAnalyzers().getDefaultIndexAnalyzer(); } return analyzer; }
if (indexAnalyzer() == null || "default".equals(indexAnalyzer().name())) { if (other.indexAnalyzer() != null && "default".equals(other.indexAnalyzer().name()) == false) { conflicts.add("mapper [" + name() + "] has different [analyzer]"); } else if (other.indexAnalyzer() == null || "default".equals(other.indexAnalyzer().name())) { conflicts.add("mapper [" + name() + "] has different [analyzer]"); } else if (indexAnalyzer().name().equals(other.indexAnalyzer().name()) == false) { conflicts.add("mapper [" + name() + "] has different [analyzer]");
/** * Fixes problems with broken analysis chains if positions and offsets are messed up that can lead to * {@link StringIndexOutOfBoundsException} in the {@link FastVectorHighlighter} */ public static WeightedFragInfo fixWeightedFragInfo(MappedFieldType fieldType, Field[] values, WeightedFragInfo fragInfo) { assert fragInfo != null : "FragInfo must not be null"; assert fieldType.name().equals(values[0].name()) : "Expected MappedFieldType for field " + values[0].name(); if (!fragInfo.getSubInfos().isEmpty() && containsBrokenAnalysis(fieldType.indexAnalyzer())) { /* This is a special case where broken analysis like WDF is used for term-vector creation at index-time * which can potentially mess up the offsets. To prevent a SAIIOBException we need to resort * the fragments based on their offsets rather than using solely the positions as it is done in * the FastVectorHighlighter. Yet, this is really a lucene problem and should be fixed in lucene rather * than in this hack... aka. "we are are working on in!" */ final List<SubInfo> subInfos = fragInfo.getSubInfos(); CollectionUtil.introSort(subInfos, new Comparator<SubInfo>() { @Override public int compare(SubInfo o1, SubInfo o2) { int startOffset = o1.getTermsOffsets().get(0).getStartOffset(); int startOffset2 = o2.getTermsOffsets().get(0).getStartOffset(); return FragmentBuilderHelper.compare(startOffset, startOffset2); } }); return new WeightedFragInfo(Math.min(fragInfo.getSubInfos().get(0).getTermsOffsets().get(0).getStartOffset(), fragInfo.getStartOffset()), fragInfo.getEndOffset(), subInfos, fragInfo.getTotalBoost()); } else { return fragInfo; } }
protected MappedFieldType(MappedFieldType ref) { super(ref); this.name = ref.name(); this.boost = ref.boost(); this.docValues = ref.hasDocValues(); this.indexAnalyzer = ref.indexAnalyzer(); this.searchAnalyzer = ref.searchAnalyzer(); this.searchQuoteAnalyzer = ref.searchQuoteAnalyzer(); this.similarity = ref.similarity(); this.nullValue = ref.nullValue(); this.nullValueAsString = ref.nullValueAsString(); this.eagerGlobalOrdinals = ref.eagerGlobalOrdinals; }
if (fieldType != null) { if (fieldType.tokenized()) { analyzer = fieldType.indexAnalyzer(); } else if (fieldType instanceof KeywordFieldMapper.KeywordFieldType) { analyzer = ((KeywordFieldMapper.KeywordFieldType) fieldType).normalizer(); if (analyzer == null) { analyzer = fieldType.indexAnalyzer();
public Analyzer apply(MappedFieldType fieldType) { return fieldType.indexAnalyzer(); } };
public MapperService(IndexSettings indexSettings, IndexAnalyzers indexAnalyzers, NamedXContentRegistry xContentRegistry, SimilarityService similarityService, MapperRegistry mapperRegistry, Supplier<QueryShardContext> queryShardContextSupplier) { super(indexSettings); this.indexAnalyzers = indexAnalyzers; this.fieldTypes = new FieldTypeLookup(); this.documentParser = new DocumentMapperParser(indexSettings, this, indexAnalyzers, xContentRegistry, similarityService, mapperRegistry, queryShardContextSupplier); this.indexAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultIndexAnalyzer(), p -> p.indexAnalyzer()); this.searchAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultSearchAnalyzer(), p -> p.searchAnalyzer()); this.searchQuoteAnalyzer = new MapperAnalyzerWrapper(indexAnalyzers.getDefaultSearchQuoteAnalyzer(), p -> p.searchQuoteAnalyzer()); this.mapperRegistry = mapperRegistry; if (indexSettings.getIndexVersionCreated().onOrAfter(Version.V_6_0_0_rc1)) { if (INDEX_MAPPER_DYNAMIC_SETTING.exists(indexSettings.getSettings())) { DEPRECATION_LOGGER.deprecated("Setting " + INDEX_MAPPER_DYNAMIC_SETTING.getKey() + " is deprecated since indices may not have more than one type anymore."); } this.dynamic = INDEX_MAPPER_DYNAMIC_DEFAULT; } else { this.dynamic = this.indexSettings.getValue(INDEX_MAPPER_DYNAMIC_SETTING); } defaultMappingSource = "{\"_default_\":{}}"; if (logger.isTraceEnabled()) { logger.trace("using dynamic[{}], default mapping source[{}]", dynamic, defaultMappingSource); } else if (logger.isDebugEnabled()) { logger.debug("using dynamic[{}]", dynamic); } }
@Override public AllFieldMapper build(BuilderContext context) { // In case the mapping overrides these // TODO: this should be an exception! it doesnt make sense to not index this field if (fieldType.indexOptions() == IndexOptions.NONE) { fieldType.setIndexOptions(Defaults.FIELD_TYPE.indexOptions()); } else { fieldType.setIndexAnalyzer(new NamedAnalyzer(fieldType.indexAnalyzer(), Defaults.POSITION_INCREMENT_GAP)); fieldType.setSearchAnalyzer(new NamedAnalyzer(fieldType.searchAnalyzer(), Defaults.POSITION_INCREMENT_GAP)); fieldType.setSearchQuoteAnalyzer(new NamedAnalyzer(fieldType.searchQuoteAnalyzer(), Defaults.POSITION_INCREMENT_GAP)); } fieldType.setTokenized(true); return new AllFieldMapper(fieldType, enabled, context.indexSettings()); } }
+ name + "] without positions enabled"); fieldType.setIndexAnalyzer(new NamedAnalyzer(fieldType.indexAnalyzer(), positionIncrementGap)); fieldType.setSearchAnalyzer(new NamedAnalyzer(fieldType.searchAnalyzer(), positionIncrementGap)); fieldType.setSearchQuoteAnalyzer(new NamedAnalyzer(fieldType.searchQuoteAnalyzer(), positionIncrementGap)); prefixFieldType.setStoreTermVectorOffsets(true); prefixFieldType.setAnalyzer(fieldType.indexAnalyzer()); prefixMapper = new PrefixFieldMapper(prefixFieldType, context.indexSettings());
protected void setupFieldType(BuilderContext context) { fieldType.setName(buildFullName(context)); if (context.indexCreatedVersion().before(Version.V_5_0_0_alpha1)) { fieldType.setOmitNorms(fieldType.omitNorms() && fieldType.boost() == 1.0f); } if (fieldType.indexAnalyzer() == null && fieldType.tokenized() == false && fieldType.indexOptions() != IndexOptions.NONE) { fieldType.setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); fieldType.setSearchAnalyzer(Lucene.KEYWORD_ANALYZER); } boolean defaultDocValues = defaultDocValues(context.indexCreatedVersion()); defaultFieldType.setHasDocValues(defaultDocValues); if (docValuesSet == false) { fieldType.setHasDocValues(defaultDocValues); } } }
private static Analyzer getAnalyzerAtField(IndexShard indexShard, String field, @Nullable Map<String, String> perFieldAnalyzer) { MapperService mapperService = indexShard.mapperService(); Analyzer analyzer; if (perFieldAnalyzer != null && perFieldAnalyzer.containsKey(field)) { analyzer = mapperService.getIndexAnalyzers().get(perFieldAnalyzer.get(field).toString()); } else { analyzer = mapperService.fullName(field).indexAnalyzer(); } if (analyzer == null) { analyzer = mapperService.getIndexAnalyzers().getDefaultIndexAnalyzer(); } return analyzer; }
private Analyzer getAnalyzerAtField(String field, @Nullable Map<String, String> perFieldAnalyzer) { MapperService mapperService = indexShard.mapperService(); Analyzer analyzer; if (perFieldAnalyzer != null && perFieldAnalyzer.containsKey(field)) { analyzer = mapperService.analysisService().analyzer(perFieldAnalyzer.get(field).toString()); } else { analyzer = mapperService.smartNameFieldType(field).indexAnalyzer(); } if (analyzer == null) { analyzer = mapperService.analysisService().defaultIndexAnalyzer(); } return analyzer; }
private Analyzer findAnalyzer(ParseContext context) { Analyzer analyzer = fieldType().indexAnalyzer(); if (analyzer == null) { analyzer = context.docMapper().mappers().indexAnalyzer(); if (analyzer == null) { // This should not happen, should we log warn it? analyzer = Lucene.STANDARD_ANALYZER; } } return analyzer; }
protected String toString(MappedFieldType ft) { return "MappedFieldType{" + "name=" + ft.name() + ", boost=" + ft.boost() + ", docValues=" + ft.hasDocValues() + ", indexAnalyzer=" + ft.indexAnalyzer() + ", searchAnalyzer=" + ft.searchAnalyzer() + ", searchQuoteAnalyzer=" + ft.searchQuoteAnalyzer() + ", similarity=" + ft.similarity() + ", eagerGlobalOrdinals=" + ft.eagerGlobalOrdinals() + ", nullValue=" + ft.nullValue() + ", nullValueAsString='" + ft.nullValueAsString() + "'" + "} " + super.toString(); }
protected MappedFieldType(MappedFieldType ref) { super(ref); this.name = ref.name(); this.boost = ref.boost(); this.docValues = ref.hasDocValues(); this.indexAnalyzer = ref.indexAnalyzer(); this.searchAnalyzer = ref.searchAnalyzer(); this.searchQuoteAnalyzer = ref.searchQuoteAnalyzer(); this.similarity = ref.similarity(); this.nullValue = ref.nullValue(); this.nullValueAsString = ref.nullValueAsString(); this.eagerGlobalOrdinals = ref.eagerGlobalOrdinals; }
protected MappedFieldType(MappedFieldType ref) { super(ref); this.name = ref.name(); this.boost = ref.boost(); this.docValues = ref.hasDocValues(); this.indexAnalyzer = ref.indexAnalyzer(); this.searchAnalyzer = ref.searchAnalyzer(); this.searchQuoteAnalyzer = ref.searchQuoteAnalyzer(); this.similarity = ref.similarity(); this.nullValue = ref.nullValue(); this.nullValueAsString = ref.nullValueAsString(); this.eagerGlobalOrdinals = ref.eagerGlobalOrdinals; }
protected void setupFieldType(BuilderContext context) { fieldType.setName(buildFullName(context)); if (context.indexCreatedVersion().before(Version.V_5_0_0_alpha1)) { fieldType.setOmitNorms(fieldType.omitNorms() && fieldType.boost() == 1.0f); } if (fieldType.indexAnalyzer() == null && fieldType.tokenized() == false && fieldType.indexOptions() != IndexOptions.NONE) { fieldType.setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); fieldType.setSearchAnalyzer(Lucene.KEYWORD_ANALYZER); } boolean defaultDocValues = defaultDocValues(context.indexCreatedVersion()); defaultFieldType.setHasDocValues(defaultDocValues); if (docValuesSet == false) { fieldType.setHasDocValues(defaultDocValues); } } }