private static ITypeTraits[] getTypeTraits(MetadataProvider metadataProvider, Dataset dataset, Index index, ARecordType recordType, ARecordType metaType) throws AlgebricksException { ITypeTraits[] primaryTypeTraits = dataset.getPrimaryTypeTraits(metadataProvider, recordType, metaType); if (index.isPrimaryIndex()) { return primaryTypeTraits; } else if (dataset.getDatasetType() == DatasetType.EXTERNAL && index.getIndexName().equals(IndexingConstants.getFilesIndexName(dataset.getDatasetName()))) { return FilesIndexDescription.EXTERNAL_FILE_INDEX_TYPE_TRAITS; int numSecondaryKeys = index.getKeyFieldNames().size(); ITypeTraitProvider typeTraitProvider = metadataProvider.getStorageComponentProvider().getTypeTraitProvider(); ITypeTraits[] secondaryTypeTraits = new ITypeTraits[numSecondaryKeys + numPrimaryKeys]; for (int i = 0; i < numSecondaryKeys; i++) { ARecordType sourceType; List<Integer> keySourceIndicators = index.getKeyFieldSourceIndicators(); if (keySourceIndicators == null || keySourceIndicators.get(i) == 0) { sourceType = recordType; sourceType = metaType; Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(i), index.getKeyFieldNames().get(i), sourceType); IAType keyType = keyTypePair.first; secondaryTypeTraits[i] = typeTraitProvider.getTypeTrait(keyType);
public Index dropIndex(Index index) { synchronized (indexes) { Map<String, Map<String, Index>> datasetMap = indexes.get(index.getDataverseName()); if (datasetMap == null) { return null; } Map<String, Index> indexMap = datasetMap.get(index.getDatasetName()); if (indexMap == null) { return null; } return indexMap.remove(index.getIndexName()); } }
List<Integer> keySourceIndicators = index.getKeyFieldSourceIndicators(); switch (index.getIndexType()) { case BTREE: case SINGLE_PARTITION_WORD_INVIX: case LENGTH_PARTITIONED_WORD_INVIX: case LENGTH_PARTITIONED_NGRAM_INVIX: return index.getKeyFieldNames().size(); case RTREE: Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0), index.getKeyFieldNames().get(0), chooseSource(keySourceIndicators, 0, recordType, metaRecordType)); IAType keyType = keyPairType.first; return NonTaggedFormatUtil.getNumDimensions(keyType.getTypeTag()) * 2; default: throw new CompilationException(ErrorCode.COMPILATION_UNKNOWN_INDEX_TYPE, index.getIndexType());
private void ensureNonPrimaryIndexDrop(Index index, SourceLocation sourceLoc) throws AlgebricksException { if (index.isPrimaryIndex()) { throw new MetadataException(ErrorCode.CANNOT_DROP_INDEX, sourceLoc, index.getIndexName(), index.getDatasetName()); } }
public static boolean isFileIndex(Index index) { return index.getIndexName().equals(IndexingConstants.getFilesIndexName(index.getDatasetName())); } }
aString.setValue(instance.getDataverseName()); stringSerde.serialize(aString, tupleBuilder.getDataOutput()); tupleBuilder.addFieldEndOffset(); aString.setValue(instance.getDatasetName()); stringSerde.serialize(aString, tupleBuilder.getDataOutput()); tupleBuilder.addFieldEndOffset(); aString.setValue(instance.getIndexName()); stringSerde.serialize(aString, tupleBuilder.getDataOutput()); tupleBuilder.addFieldEndOffset(); aString.setValue(instance.getDataverseName()); stringSerde.serialize(aString, fieldValue.getDataOutput()); recordBuilder.addField(MetadataRecordTypes.INDEX_ARECORD_DATAVERSENAME_FIELD_INDEX, fieldValue); aString.setValue(instance.getDatasetName()); stringSerde.serialize(aString, fieldValue.getDataOutput()); recordBuilder.addField(MetadataRecordTypes.INDEX_ARECORD_DATASETNAME_FIELD_INDEX, fieldValue); aString.setValue(instance.getIndexName()); stringSerde.serialize(aString, fieldValue.getDataOutput()); recordBuilder.addField(MetadataRecordTypes.INDEX_ARECORD_INDEXNAME_FIELD_INDEX, fieldValue); aString.setValue(instance.getIndexType().toString()); stringSerde.serialize(aString, fieldValue.getDataOutput()); recordBuilder.addField(MetadataRecordTypes.INDEX_ARECORD_INDEXSTRUCTURE_FIELD_INDEX, fieldValue); List<List<String>> searchKey = instance.getKeyFieldNames(); for (List<String> field : searchKey) { listBuilder.reset(stringList);
@Override protected void setSecondaryRecDescAndComparators() throws AlgebricksException { int numSecondaryKeys = index.getKeyFieldNames().size(); secondaryFieldAccessEvalFactories = new IScalarEvaluatorFactory[numSecondaryKeys + numFilterFields]; secondaryComparatorFactories = new IBinaryComparatorFactory[numSecondaryKeys + numPrimaryKeys]; boolean isOverridingKeyTypes = index.isOverridingKeyFieldTypes(); for (int i = 0; i < numSecondaryKeys; i++) { ARecordType sourceType; int sourceColumn; List<Integer> keySourceIndicators = index.getKeyFieldSourceIndicators(); if (keySourceIndicators == null || keySourceIndicators.get(i) == 0) { sourceType = itemType; index.getKeyFieldNames().get(i), sourceColumn, sourceLoc); Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(i), index.getKeyFieldNames().get(i), sourceType); IAType keyType = keyTypePair.first; anySecondaryKeyIsNullable = anySecondaryKeyIsNullable || keyTypePair.second; metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory( metadataProvider.getFunctionManager(), itemType, filterFieldName, recordColumn, sourceLoc); Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType); IAType type = keyTypePair.first; ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type);
new Index(dataverseName, datasetName, indexName, index.getIndexType(), index.getKeyFieldNames(), index.getKeyFieldSourceIndicators(), index.getKeyFieldTypes(), index.isOverridingKeyFieldTypes(), index.isEnforced(), index.isPrimaryIndex(), MetadataUtil.PENDING_DROP_OP)); externalIndex.getIndexName()); MetadataManager.INSTANCE.addIndex(mdTxnCtx, new Index(dataverseName, datasetName, externalIndex.getIndexName(), externalIndex.getIndexType(), externalIndex.getKeyFieldNames(), externalIndex.getKeyFieldSourceIndicators(), index.getKeyFieldTypes(), index.isOverridingKeyFieldTypes(), index.isEnforced(), externalIndex.isPrimaryIndex(), MetadataUtil.PENDING_DROP_OP)); new Index(dataverseName, datasetName, indexName, index.getIndexType(), index.getKeyFieldNames(), index.getKeyFieldSourceIndicators(), index.getKeyFieldTypes(), index.isOverridingKeyFieldTypes(), index.isEnforced(), index.isPrimaryIndex(), MetadataUtil.PENDING_DROP_OP));
boolean filesIndexReplicated = false; try { index.setPendingOp(MetadataUtil.PENDING_ADD_OP); if (ds.getDatasetType() == DatasetType.INTERNAL) { validateDatasetState(metadataProvider, ds, sourceLoc); if (!ExternalIndexingOperations.isValidIndexName(index.getDatasetName(), index.getIndexName())) { throw new CompilationException(ErrorCode.COMPILATION_ERROR, sourceLoc, "external dataset index name is invalid"); index.getDataverseName(), index.getDatasetName(), IndexingConstants.getFilesIndexName(index.getDatasetName())); firstExternalDatasetIndex = filesIndex == null; index.getDataverseName(), index.getDatasetName(), IndexingConstants.getFilesIndexName(index.getDatasetName())); if (filesIndex != null) { ExternalDatasetsRegistry.INSTANCE.buildIndexEnd(ds, firstExternalDatasetIndex); filesIndex = new Index(index.getDataverseName(), index.getDatasetName(), IndexingConstants.getFilesIndexName(index.getDatasetName()), IndexType.BTREE, ExternalIndexingOperations.FILE_INDEX_FIELD_NAMES, null, ExternalIndexingOperations.FILE_INDEX_FIELD_TYPES, false, false, false, if (index.isEnforced()) { List<Index> indexes = MetadataManager.INSTANCE.getDatasetIndexes( metadataProvider.getMetadataTxnContext(), index.getDataverseName(), index.getDatasetName()); for (Index existingIndex : indexes) { if (existingIndex.getKeyFieldNames().equals(index.getKeyFieldNames())
Collections.sort(indexes, (o1, o2) -> o1.getIndexType().ordinal() - o2.getIndexType().ordinal()); if (!index.isSecondaryIndex()) { continue; List<List<String>> secondaryKeyFields = index.getKeyFieldNames(); List<IAType> secondaryKeyTypes = index.getKeyFieldTypes(); List<LogicalVariable> secondaryKeyVars = new ArrayList<>(); List<Mutable<ILogicalExpression>> secondaryExpressions = new ArrayList<>(); IndexFieldId indexFieldId = new IndexFieldId(index.getKeyFieldSourceIndicators().get(i), secondaryKeyFields.get(i), secondaryKeyTypes.get(i).getTypeTag()); LogicalVariable skVar = fieldVarsForNewRecord.get(indexFieldId); if (index.getIndexType() != IndexType.RTREE) { index.isOverridingKeyFieldTypes(), sourceLoc); DataSourceIndex dataSourceIndex = new DataSourceIndex(index, dataverseName, datasetName, mp); if (index.getIndexType() != IndexType.BTREE && primaryIndexModificationOp.isBulkload()) { boolean isPartitioned = index.getIndexType() == IndexType.LENGTH_PARTITIONED_WORD_INVIX || index.getIndexType() == IndexType.LENGTH_PARTITIONED_NGRAM_INVIX; Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType( index.getKeyFieldTypes().get(0), secondaryKeyFields.get(0), recType); secondaryKeyType = keyPairType.first; Pair<IAType, Boolean> keyPairType = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0),
getSplitProviderAndConstraints(dataset, secondaryIndex.getIndexName()); int[] primaryKeyFields = new int[numPrimaryKeys]; for (int i = 0; i < numPrimaryKeys; i++) { if (isIndexOnlyPlan) { ARecordType recType = (ARecordType) findType(dataset.getItemTypeDataverseName(), dataset.getItemTypeName()); List<List<String>> secondaryKeyFields = secondaryIndex.getKeyFieldNames(); List<IAType> secondaryKeyTypes = secondaryIndex.getKeyFieldTypes(); Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(secondaryKeyTypes.get(0), secondaryKeyFields.get(0), recType); IAType keyType = keyTypePair.first; int numDimensions = NonTaggedFormatUtil.getNumDimensions(keyType.getTypeTag());
@Override protected void setSecondaryRecDescAndComparators() throws AlgebricksException { int numSecondaryKeys = index.getKeyFieldNames().size(); IndexType indexType = index.getIndexType(); boolean isOverridingKeyFieldTypes = index.isOverridingKeyFieldTypes(); secondaryFieldAccessEvalFactories[0] = metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory( metadataProvider.getFunctionManager(), isOverridingKeyFieldTypes ? enforcedItemType : itemType, index.getKeyFieldNames().get(0), numPrimaryKeys, sourceLoc); Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0), index.getKeyFieldNames().get(0), itemType); secondaryKeyType = keyTypePair.first; anySecondaryKeyIsNullable = anySecondaryKeyIsNullable || keyTypePair.second; .getFieldAccessEvaluatorFactory(metadataProvider.getFunctionManager(), itemType, filterFieldName, numPrimaryKeys, sourceLoc); Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType); IAType type = keyTypePair.first; ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type); index.getGramLength());
SourceLocation sourceLoc = currentTop.getSourceLocation(); for (Index index : indexes) { if (index.isPrimaryIndex()) { continue; List<IAType> skTypes = index.getKeyFieldTypes(); List<List<String>> skNames = index.getKeyFieldNames(); List<Integer> indicators = index.getKeyFieldSourceIndicators(); for (int i = 0; i < index.getKeyFieldNames().size(); i++) { IndexFieldId indexFieldId = new IndexFieldId(indicators.get(i), skNames.get(i), skTypes.get(i).getTypeTag()); index.isEnforced() ? BuiltinFunctions.CAST_TYPE : BuiltinFunctions.CAST_TYPE_LAX)); theFieldAccessFunc.setSourceLocation(sourceLoc);
if (index.getKeyFieldNames().contains(fieldName) && index.getPendingOp() == MetadataUtil.PENDING_NO_OP) { indexCandidates.add(index); boolean isFieldTypeUnknown = fieldType == BuiltinType.AMISSING || fieldType == BuiltinType.ANY; if (isFieldTypeUnknown && (!index.isOverridingKeyFieldTypes() || index.isEnforced())) { IAType indexedType = index.getKeyFieldTypes().get(index.getKeyFieldNames().indexOf(fieldName)); optFuncExpr.setFieldType(varIdx, indexedType);
if (dataset.getDatasetType() == DatasetType.EXTERNAL || chosenIndex.isPrimaryIndex() || chosenIndex.isOverridingKeyFieldTypes() || chosenIndex.isEnforced() || isInvertedIndex(chosenIndex) || noIndexOnlyPlanOption) { indexOnlyPlanInfo.setFirst(false); List<List<String>> chosenIndexFieldNames = chosenIndex.getKeyFieldNames(); List<LogicalVariable> chosenIndexVars = new ArrayList<>(); if (chosenIndex.getKeyFieldNames().size() > 1 && indexApplicableVarFoundCount > 1) { requireVerificationAfterSIdxSearch = true; if (chosenIndex.getIndexType() == IndexType.RTREE) { boolean rTreeCheck = checkRTreeSpecificIdxOnlyCondition(probeSubTree, indexSubTree, chosenIndex, analysisCtx, matchedFuncExprs, liveVarsAfterSelJoinOp, indexOnlyPlanInfo);
private boolean isContainsFuncJoinOptimizable(Index index, IOptimizableFuncExpr optFuncExpr) { if (index.isEnforced()) { return isContainsFuncCompatible(index.getKeyFieldTypes().get(0).getTypeTag(), index.getIndexType()); } else { return isContainsFuncCompatible(optFuncExpr.getFieldType(0).getTypeTag(), index.getIndexType()); } }
private static int[] getBloomFilterFields(Dataset dataset, Index index) throws AlgebricksException { if (index.isPrimaryIndex()) { return dataset.getPrimaryBloomFilterFields(); } else if (dataset.getDatasetType() == DatasetType.EXTERNAL) { if (index.getIndexName().equals(IndexingConstants.getFilesIndexName(dataset.getDatasetName()))) { return FilesIndexDescription.BLOOM_FILTER_FIELDS; } else { return new int[] { index.getKeyFieldNames().size() }; } } int numKeys = index.getKeyFieldNames().size(); int[] bloomFilterKeyFields = new int[numKeys]; for (int i = 0; i < numKeys; i++) { bloomFilterKeyFields[i] = i; } return bloomFilterKeyFields; } }
public SecondaryIndexInfo(PrimaryIndexInfo primaryIndexInfo, Index secondaryIndex) { this.primaryIndexInfo = primaryIndexInfo; this.secondaryIndex = secondaryIndex; List<String> nodes = Collections.singletonList(ExecutionTestUtil.integrationUtil.ncs[0].getId()); CcApplicationContext appCtx = (CcApplicationContext) ExecutionTestUtil.integrationUtil.cc.getApplicationContext(); FileSplit[] splits = SplitsAndConstraintsUtil.getIndexSplits(appCtx.getClusterStateManager(), primaryIndexInfo.dataset, secondaryIndex.getIndexName(), nodes); fileSplitProvider = new ConstantFileSplitProvider(splits); secondaryIndexTypeTraits = createSecondaryIndexTypeTraits(primaryIndexInfo.recordType, primaryIndexInfo.metaType, primaryIndexInfo.primaryKeyTypes, secondaryIndex.getKeyFieldTypes().toArray(new IAType[secondaryIndex.getKeyFieldTypes().size()])); secondaryIndexSerdes = createSecondaryIndexSerdes(primaryIndexInfo.recordType, primaryIndexInfo.metaType, primaryIndexInfo.primaryKeyTypes, secondaryIndex.getKeyFieldTypes().toArray(new IAType[secondaryIndex.getKeyFieldTypes().size()])); rDesc = new RecordDescriptor(secondaryIndexSerdes, secondaryIndexTypeTraits); insertFieldsPermutations = new int[secondaryIndexTypeTraits.length]; for (int i = 0; i < insertFieldsPermutations.length; i++) { insertFieldsPermutations[i] = i; } primaryKeyIndexes = new int[primaryIndexInfo.primaryKeyIndexes.length]; for (int i = 0; i < primaryKeyIndexes.length; i++) { primaryKeyIndexes[i] = i + secondaryIndex.getKeyFieldNames().size(); } }
if (chosenIndex.isEnforced() && realTypeConvertedToIntegerType) { return true; if (chosenIndex.isOverridingKeyFieldTypes() && !chosenIndex.isEnforced()) { IAType indexedKeyType = chosenIndex.getKeyFieldTypes().get(keyPos); if (NonTaggedFormatUtil.isOptional(indexedKeyType)) { indexedKeyType = ((AUnionType) indexedKeyType).getActualType();
public boolean isSecondaryIndex() { return !isPrimaryIndex(); }