public static IBinaryTokenizerFactory getBinaryTokenizerFactory(SearchModifierType searchModifierType, ATypeTag searchKeyType, Index index) throws AlgebricksException { switch (index.getIndexType()) { case SINGLE_PARTITION_WORD_INVIX: case LENGTH_PARTITIONED_WORD_INVIX: { return BinaryTokenizerFactoryProvider.INSTANCE.getWordTokenizerFactory(searchKeyType, false, false); } case SINGLE_PARTITION_NGRAM_INVIX: case LENGTH_PARTITIONED_NGRAM_INVIX: { // Make sure not to use pre- and postfixing for conjunctive searches. boolean prePost = (searchModifierType == SearchModifierType.CONJUNCTIVE || searchModifierType == SearchModifierType.CONJUNCTIVE_EDIT_DISTANCE) ? false : true; return BinaryTokenizerFactoryProvider.INSTANCE.getNGramTokenizerFactory(searchKeyType, index.getGramLength(), prePost, false); } default: { throw new CompilationException(ErrorCode.NO_TOKENIZER_FOR_TYPE, index.getIndexType()); } } }
index.getGramLength());
mergeThreshold = (astr.getStringValue().length() - index.getGramLength() + 1) - edThresh.getIntegerValue() * index.getGramLength(); } else { mergeThreshold = (astr.getStringValue().length() + index.getGramLength() - 1) - edThresh.getIntegerValue() * index.getGramLength();
private boolean isContainsFuncSelectOptimizable(Index index, IOptimizableFuncExpr optFuncExpr) { AsterixConstantValue strConstVal = (AsterixConstantValue) ((ConstantExpression) optFuncExpr.getConstantExpr(0)).getValue(); IAObject strObj = strConstVal.getObject(); ATypeTag typeTag = strObj.getType().getTypeTag(); if (!isContainsFuncCompatible(typeTag, index.getIndexType())) { return false; } // Check that the constant search string has at least gramLength characters. if (strObj.getType().getTypeTag() == ATypeTag.STRING) { AString astr = (AString) strObj; if (astr.getStringValue().length() >= index.getGramLength()) { return true; } } return false; }
return new EditDistanceSearchModifierFactory(index.getGramLength(), edThresh); } else { return new ConjunctiveEditDistanceSearchModifierFactory(index.getGramLength(), edThresh);
AccessMethodUtils.createInt32Constant(chosenIndex.getGramLength()))); boolean usePrePost = optFuncExpr.containsPartialField() ? false : true; isFilterableArgs.add(
secondaryKeyType.getTypeTag(), indexType, secondaryIndex.getGramLength());
index.getGramLength());
index.getGramLength());
if (instance.getGramLength() > 0) { fieldValue.reset(); nameValue.reset(); aString.setValue(GRAM_LENGTH_FIELD_NAME); stringSerde.serialize(aString, nameValue.getDataOutput()); intSerde.serialize(new AInt32(instance.getGramLength()), fieldValue.getDataOutput()); recordBuilder.addField(nameValue, fieldValue);