/** * Finds ngram of cerain order in specified range and reads it's probability. * Range contains ngram successors after function execution. * If ngram is not found, range will be invalid. * @param wordId - word id to look for * @param orderMinusTwo - order of ngram minus two * @param range - range to look in, contains ngram successors after function execution * @param quant - quantation object to decode compressed probability stored in trie * @return probability of ngram */ public float readNgramProb(int wordId, int orderMinusTwo, TrieRange range, NgramTrieQuant quant) { int ptr; NgramSet ngram = getNgram(orderMinusTwo); if ((ptr = findNgram(ngram, wordId, range)) < 0) return 0.0f; return quant.readProb(bitArr, ngram.memPtr, ngram.getNgramWeightsOffset(ptr), orderMinusTwo); }
/** * Searches ngram index for given wordId in provided range */ private int uniformFind(NgramSet ngram, TrieRange range, int wordId) { TrieRange vocabRange = new TrieRange(0, ngram.maxVocab); while (range.getWidth() > 1) { int pivot = range.begin + 1 + calculatePivot(wordId - vocabRange.begin, vocabRange.getWidth(), range.getWidth() - 1); int mid = ngram.readNgramWord(pivot); if (mid < wordId) { range.begin = pivot; vocabRange.begin = mid; } else if (mid > wordId){ range.end = pivot; vocabRange.end = mid; } else { return pivot; } } return -1; }
/** * Finds ngram of cerain order in specified range and reads it's backoff. * Range contains ngram successors after function execution. * If ngram is not found, range will be invalid. * @param wordId - word id to look for * @param orderMinusTwo - order of ngram minus two * @param range - range to look in, contains ngram successors after function execution * @param quant - quantation object to decode compressed backoff stored in trie * @return backoff of ngram */ public float readNgramBackoff(int wordId, int orderMinusTwo, TrieRange range, NgramTrieQuant quant) { int ptr; NgramSet ngram = getNgram(orderMinusTwo); if ((ptr = findNgram(ngram, wordId, range)) < 0) return 0.0f; return quant.readBackoff(bitArr, ngram.memPtr, ngram.getNgramWeightsOffset(ptr), orderMinusTwo); }
/** * Searches ngram index for given wordId in provided range */ private int uniformFind(NgramSet ngram, TrieRange range, int wordId) { TrieRange vocabRange = new TrieRange(0, ngram.maxVocab); while (range.getWidth() > 1) { int pivot = range.begin + 1 + calculatePivot(wordId - vocabRange.begin, vocabRange.getWidth(), range.getWidth() - 1); int mid = ngram.readNgramWord(pivot); if (mid < wordId) { range.begin = pivot; vocabRange.begin = mid; } else if (mid > wordId){ range.end = pivot; vocabRange.end = mid; } else { return pivot; } } return -1; }
/** * Finds ngram of cerain order in specified range and reads it's probability. * Range contains ngram successors after function execution. * If ngram is not found, range will be invalid. * @param wordId - word id to look for * @param orderMinusTwo - order of ngram minus two * @param range - range to look in, contains ngram successors after function execution * @param quant - quantation object to decode compressed probability stored in trie * @return probability of ngram */ public float readNgramProb(int wordId, int orderMinusTwo, TrieRange range, NgramTrieQuant quant) { int ptr; NgramSet ngram = getNgram(orderMinusTwo); if ((ptr = findNgram(ngram, wordId, range)) < 0) return 0.0f; return quant.readProb(bitArr, ngram.memPtr, ngram.getNgramWeightsOffset(ptr), orderMinusTwo); }
/** * Finds ngram of cerain order in specified range and reads it's backoff. * Range contains ngram successors after function execution. * If ngram is not found, range will be invalid. * @param wordId - word id to look for * @param orderMinusTwo - order of ngram minus two * @param range - range to look in, contains ngram successors after function execution * @param quant - quantation object to decode compressed backoff stored in trie * @return backoff of ngram */ public float readNgramBackoff(int wordId, int orderMinusTwo, TrieRange range, NgramTrieQuant quant) { int ptr; NgramSet ngram = getNgram(orderMinusTwo); if ((ptr = findNgram(ngram, wordId, range)) < 0) return 0.0f; return quant.readBackoff(bitArr, ngram.memPtr, ngram.getNgramWeightsOffset(ptr), orderMinusTwo); }