protected AbstractIndexOrdinalsFieldData(Index index, Settings indexSettings, Names fieldNames, FieldDataType fieldDataType, IndexFieldDataCache cache, CircuitBreakerService breakerService) { super(index, indexSettings, fieldNames, fieldDataType, cache); final Map<String, Settings> groups = fieldDataType.getSettings().getGroups("filter"); frequency = groups.get("frequency"); regex = groups.get("regex"); this.breakerService = breakerService; }
/** * Gets a memory storage hint that should be honored if possible but is not mandatory */ public static MemoryStorageFormat getMemoryStorageHint(FieldDataType fieldDataType) { // backwards compatibility String s = fieldDataType.getSettings().get("ordinals"); if (s != null) { return "always".equals(s) ? MemoryStorageFormat.ORDINALS : null; } return MemoryStorageFormat.fromString(fieldDataType.getSettings().get(SETTING_MEMORY_STORAGE_HINT)); } }
@Override public Builder fieldDataSettings(Settings fieldDataSettings) { Settings settings = Settings.builder().put(childJoinFieldType.fieldDataType().getSettings()).put(fieldDataSettings).build(); childJoinFieldType.setFieldDataType(new FieldDataType(childJoinFieldType.fieldDataType().getType(), settings)); return this; }
@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { if (!active()) { return builder; } boolean includeDefaults = params.paramAsBoolean("include_defaults", false); builder.startObject(CONTENT_TYPE); builder.field("type", parentType); if (includeDefaults || joinFieldHasCustomFieldDataSettings()) { builder.field("fielddata", (Map) childJoinFieldType.fieldDataType().getSettings().getAsMap()); } builder.endObject(); return builder; }
@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { if (indexCreatedBefore2x == false) { return builder; } boolean includeDefaults = params.paramAsBoolean("include_defaults", false); // if defaults, don't output if (!includeDefaults && hasCustomFieldDataSettings() == false) { return builder; } builder.startObject(CONTENT_TYPE); if (includeDefaults || hasCustomFieldDataSettings()) { builder.field("fielddata", (Map) fieldType().fieldDataType().getSettings().getAsMap()); } builder.endObject(); return builder; }
final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat( FilterSettingFields.ACCEPTABLE_TRANSIENT_OVERHEAD_RATIO, OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO); fieldDataType.getSettings().getAsDouble(FilterSettingFields.FREQUENCY_MIN, 0d) != 0d || fieldDataType.getSettings().getAsDouble(FilterSettingFields.FREQUENCY_MAX, 0d) != 0d || fieldDataType.getSettings().getAsDouble(FilterSettingFields.FREQUENCY_MIN_SEGMENT_SIZE, 0d) != 0d || fieldDataType.getSettings().get(FilterSettingFields.REGEX_PATTERN) != null) { if (logger.isTraceEnabled()) { logger.trace("Filter exists, can't circuit break normally, using RamAccountingTermsEnum");
@Override public IndexFieldData<?> build(Index index, Settings indexSettings, MappedFieldType fieldType, IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { // Ignore Circuit Breaker final Names fieldNames = fieldType.names(); final Settings fdSettings = fieldType.fieldDataType().getSettings(); final Map<String, Settings> filter = fdSettings.getGroups("filter"); if (filter != null && !filter.isEmpty()) { throw new IllegalArgumentException("Doc values field data doesn't support filters [" + fieldNames.fullName() + "]"); } if (BINARY_INDEX_FIELD_NAMES.contains(fieldNames.indexName())) { assert numericType == null; return new BinaryDVIndexFieldData(index, fieldNames, fieldType.fieldDataType()); } else if (numericType != null) { if (TimestampFieldMapper.NAME.equals(fieldNames.indexName()) || Version.indexCreated(indexSettings).onOrAfter(Version.V_1_4_0_Beta1)) { return new SortedNumericDVIndexFieldData(index, fieldNames, numericType, fieldType.fieldDataType()); } else { // prior to ES 1.4: multi-valued numerics were boxed inside a byte[] as BINARY return new BinaryDVNumericIndexFieldData(index, fieldNames, numericType, fieldType.fieldDataType()); } } else { return new SortedSetDVOrdinalsIndexFieldData(index, cache, indexSettings, fieldNames, breakerService, fieldType.fieldDataType()); } }
@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { boolean includeDefaults = params.paramAsBoolean("include_defaults", false); // if all defaults, no need to write it at all if (!includeDefaults && fieldType().stored() == Defaults.FIELD_TYPE.stored() && enabledState == Defaults.ENABLED_STATE && hasCustomFieldDataSettings() == false) { return builder; } builder.startObject(CONTENT_TYPE); if (indexCreatedBefore2x && (includeDefaults || fieldType().stored() != Defaults.FIELD_TYPE.stored())) { builder.field("store", fieldType().stored()); } if (includeDefaults || enabledState != Defaults.ENABLED_STATE) { builder.field("enabled", enabledState.enabled); } if (indexCreatedBefore2x && (includeDefaults || hasCustomFieldDataSettings())) { builder.field("fielddata", (Map) fieldType().fieldDataType().getSettings().getAsMap()); } builder.endObject(); return builder; }
String cacheType = type.getSettings().get("cache", indexSettings.get(FIELDDATA_CACHE_KEY, FIELDDATA_CACHE_VALUE_NODE)); if (FIELDDATA_CACHE_VALUE_NODE.equals(cacheType)) { cache = indicesFieldDataCache.buildIndexFieldDataCache(listener, index, fieldNames, type);
protected void setupFieldType(BuilderContext context) { fieldType.setNames(buildNames(context)); if (fieldType.indexAnalyzer() == null && fieldType.tokenized() == false && fieldType.indexOptions() != IndexOptions.NONE) { fieldType.setIndexAnalyzer(Lucene.KEYWORD_ANALYZER); fieldType.setSearchAnalyzer(Lucene.KEYWORD_ANALYZER); } if (fieldDataSettings != null) { Settings settings = Settings.builder().put(fieldType.fieldDataType().getSettings()).put(fieldDataSettings).build(); fieldType.setFieldDataType(new FieldDataType(fieldType.fieldDataType().getType(), settings)); } boolean defaultDocValues = false; // pre 2.0 if (context.indexCreatedVersion().onOrAfter(Version.V_2_0_0_beta1)) { defaultDocValues = fieldType.tokenized() == false && fieldType.indexOptions() != IndexOptions.NONE; } // backcompat for "fielddata: format: docvalues" for now... boolean fieldDataDocValues = fieldType.fieldDataType() != null && FieldDataType.DOC_VALUES_FORMAT_VALUE.equals(fieldType.fieldDataType().getFormat(context.indexSettings())); if (fieldDataDocValues && docValuesSet && fieldType.hasDocValues() == false) { // this forces the doc_values setting to be written, so fielddata does not mask the original setting defaultDocValues = true; } defaultFieldType.setHasDocValues(defaultDocValues); if (docValuesSet == false) { fieldType.setHasDocValues(defaultDocValues || fieldDataDocValues); } } }
builder.field("fielddata", fieldType().fieldDataType().getSettings().getAsMap());
@Override public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { if (indexCreatedBefore2x == false) { return builder; } boolean includeDefaults = params.paramAsBoolean("include_defaults", false); // if all are defaults, no sense to write it at all if (!includeDefaults && fieldType().stored() == Defaults.FIELD_TYPE.stored() && fieldType().indexOptions() == Defaults.FIELD_TYPE.indexOptions() && path == Defaults.PATH && hasCustomFieldDataSettings() == false) { return builder; } builder.startObject(CONTENT_TYPE); if (includeDefaults || fieldType().stored() != Defaults.FIELD_TYPE.stored()) { builder.field("store", fieldType().stored()); } if (includeDefaults || fieldType().indexOptions() != Defaults.FIELD_TYPE.indexOptions()) { builder.field("index", indexTokenizeOptionToString(fieldType().indexOptions() != IndexOptions.NONE, fieldType().tokenized())); } if (includeDefaults || path != Defaults.PATH) { builder.field("path", path); } if (includeDefaults || hasCustomFieldDataSettings()) { builder.field("fielddata", (Map) fieldType().fieldDataType().getSettings().getAsMap()); } builder.endObject(); return builder; }
final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat( "acceptable_transient_overhead_ratio", OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO ); for (ObjectObjectCursor<String, TypeBuilder> cursor : typeBuilders) { PagedBytes.Reader bytesReader = cursor.value.bytes.freeze(true); final Ordinals ordinals = cursor.value.builder.build(fieldDataType.getSettings());
AtomicGeoPointFieldData data) throws Exception { LongArray indexedPoints = BigArrays.NON_RECYCLING_INSTANCE.newLongArray(128); final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat("acceptable_transient_overhead_ratio", OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO); boolean success = false; Ordinals build = builder.build(fieldDataType.getSettings()); RandomAccessOrds ordinals = build.ordinals(); if (!(FieldData.isMultiValued(ordinals) || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings
numTerms = -1; final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat( FilterSettingFields.ACCEPTABLE_TRANSIENT_OVERHEAD_RATIO, OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO); final Ordinals ordinals = builder.build(fieldDataType.getSettings());
DoubleArray lat = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(128); DoubleArray lon = BigArrays.NON_RECYCLING_INSTANCE.newDoubleArray(128); final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat("acceptable_transient_overhead_ratio", OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO); boolean success = false; try (OrdinalsBuilder builder = new OrdinalsBuilder(terms.size(), reader.maxDoc(), acceptableTransientOverheadRatio)) { lon = BigArrays.NON_RECYCLING_INSTANCE.resize(lon, numTerms); Ordinals build = builder.build(fieldDataType.getSettings()); RandomAccessOrds ordinals = build.ordinals(); if (!(FieldData.isMultiValued(ordinals) || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS)) {
final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat("acceptable_transient_overhead_ratio", OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO); TermsEnum termsEnum = estimator.beforeLoad(terms); assert !getNumericType().isFloatingPoint(); final Ordinals build = builder.build(fieldDataType.getSettings()); CommonSettings.MemoryStorageFormat formatHint = CommonSettings.getMemoryStorageHint(fieldDataType); final float acceptableOverheadRatio = fieldDataType.getSettings().getAsFloat("acceptable_overhead_ratio", PackedInts.DEFAULT); final int pageSize = fieldDataType.getSettings().getAsInt("single_value_page_size", 1024);
final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat("acceptable_transient_overhead_ratio", OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO); boolean success = false; try (OrdinalsBuilder builder = new OrdinalsBuilder(reader.maxDoc(), acceptableTransientOverheadRatio)) { final Ordinals build = builder.build(fieldDataType.getSettings()); RandomAccessOrds ordinals = build.ordinals(); if (FieldData.isMultiValued(ordinals) || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
final float acceptableTransientOverheadRatio = fieldDataType.getSettings().getAsFloat("acceptable_transient_overhead_ratio", OrdinalsBuilder.DEFAULT_ACCEPTABLE_OVERHEAD_RATIO); boolean success = false; try (OrdinalsBuilder builder = new OrdinalsBuilder(reader.maxDoc(), acceptableTransientOverheadRatio)) { final Ordinals build = builder.build(fieldDataType.getSettings()); RandomAccessOrds ordinals = build.ordinals(); if (FieldData.isMultiValued(ordinals) || CommonSettings.getMemoryStorageHint(fieldDataType) == CommonSettings.MemoryStorageFormat.ORDINALS) {
builder.field("fielddata", fieldType().fieldDataType().getSettings().getAsMap());