/** * Returns the NGramProbability of the nth follower. * * @param nthFollower which follower * @return the NGramProbability of the nth follower */ @Override public NGramProbability getNGramProbability(int nthFollower) { int nthPosition = 0, wordID = 0, probID = 0, backoffID = 0, firstNGram = 0; nthPosition = nthFollower * LargeNGramModel.BYTES_PER_NMAXGRAM * ((is32bits()) ? 4 : 2); setPosition(nthPosition); wordID = readBytesAsInt(); probID = readBytesAsInt(); return (new NGramProbability(nthFollower, wordID, probID, backoffID, firstNGram)); } }
/** * Returns the index of the first NGram entry of the given N-1Gram * * @param nMinus1Gram * the N-1Gram which first NGram entry we're looking for * @param firstNMinus1GramEntry * the index of the first N-1Gram entry of the N-1Gram in * question * @param n * the order of the NGram * @return the index of the first NGram entry of the given N-1Gram */ private int getFirstNGramEntry(NGramProbability nMinus1Gram, int firstNMinus1GramEntry, int n) { int firstNGramEntry = ngramSegmentTable[n - 1][(firstNMinus1GramEntry + nMinus1Gram .getWhichFollower()) >> loader.getLogNGramSegmentSize()] + nMinus1Gram.getFirstNPlus1GramEntry(); return firstNGramEntry; }
/** * Retrieves the bigram probability for the two given words * * @param word1 * the first word of the bigram * @param word2 * the second word of the bigram * @return the log probability */ private float getBigramProb(int word1, int word2) { NGramBuffer bigram = getBigramBuffer(word1); NGramProbability bigramProbability = bigram.findNGram(word2); return ngramProbTable[1][bigramProbability.getProbabilityID()]; }
private Float getNGramProbability(WordSequence wordSequence) { int numberWords = wordSequence.size(); Word firstWord = wordSequence.getWord(0); if (loader.getNumberNGrams(numberWords) == 0 || !hasUnigram(firstWord)) return getNGramProbability(wordSequence.getNewest()); if (numberWords < 2) { return getUnigramProbability(wordSequence); } NGramProbability nGProbability = findNGram(wordSequence); if (nGProbability != null) { return ngramProbTable[numberWords - 1][nGProbability .getProbabilityID()]; } if (numberWords == 2) { UnigramProbability unigramProb = getUnigram(firstWord); UnigramProbability unigramProb1 = getUnigram(wordSequence .getWord(1)); return unigramProb.getLogBackoff() + unigramProb1.getLogProbability(); } NGramProbability nMinus1Gram = findNGram(wordSequence.getOldest()); if (nMinus1Gram != null) { return ngramBackoffTable[numberWords - 1][nMinus1Gram .getBackoffID()] + getProbability(wordSequence.getNewest()); } return getProbability(wordSequence.getNewest()); }
float logbgprob = ngramProbTable[1][bgProb.getProbabilityID()]; float smearTerm; NGramProbability bgProb = bigram.getNGramProbability(j); float logbgbackoff = ngramBackoffTable[2][bgProb.getBackoffID()]; double bgbackoff = logMath.logToLinear(logbgbackoff); int k = bigram.getWordID(j);
private Float getNGramProbability(WordSequence wordSequence) { int numberWords = wordSequence.size(); Word firstWord = wordSequence.getWord(0); if (loader.getNumberNGrams(numberWords) == 0 || !hasUnigram(firstWord)) return getNGramProbability(wordSequence.getNewest()); if (numberWords < 2) { return getUnigramProbability(wordSequence); } NGramProbability nGProbability = findNGram(wordSequence); if (nGProbability != null) { return ngramProbTable[numberWords - 1][nGProbability .getProbabilityID()]; } if (numberWords == 2) { UnigramProbability unigramProb = getUnigram(firstWord); UnigramProbability unigramProb1 = getUnigram(wordSequence .getWord(1)); return unigramProb.getLogBackoff() + unigramProb1.getLogProbability(); } NGramProbability nMinus1Gram = findNGram(wordSequence.getOldest()); if (nMinus1Gram != null) { return ngramBackoffTable[numberWords - 1][nMinus1Gram .getBackoffID()] + getProbability(wordSequence.getNewest()); } return getProbability(wordSequence.getNewest()); }
/** * Returns the index of the first NGram entry of the given N-1Gram * * @param nMinus1Gram * the N-1Gram which first NGram entry we're looking for * @param firstNMinus1GramEntry * the index of the first N-1Gram entry of the N-1Gram in * question * @param n * the order of the NGram * @return the index of the first NGram entry of the given N-1Gram */ private int getFirstNGramEntry(NGramProbability nMinus1Gram, int firstNMinus1GramEntry, int n) { int firstNGramEntry = ngramSegmentTable[n - 1][(firstNMinus1GramEntry + nMinus1Gram .getWhichFollower()) >> loader.getLogNGramSegmentSize()] + nMinus1Gram.getFirstNPlus1GramEntry(); return firstNGramEntry; }
/** * Returns the NGramProbability of the nth follower. * * @param nthFollower which follower * @return the NGramProbability of the nth follower */ public NGramProbability getNGramProbability(int nthFollower) { int nthPosition = 0, wordID = 0, probID = 0, backoffID = 0, firstNGram = 0; nthPosition = nthFollower * LargeNGramModel.BYTES_PER_NGRAM * ((is32bits) ? 4 : 2); setPosition(nthPosition); wordID = readBytesAsInt(); probID = readBytesAsInt(); backoffID = readBytesAsInt(); firstNGram = readBytesAsInt(); return (new NGramProbability(nthFollower, wordID, probID, backoffID, firstNGram)); } }
/** * Retrieves the bigram probability for the two given words * * @param word1 * the first word of the bigram * @param word2 * the second word of the bigram * @return the log probability */ private float getBigramProb(int word1, int word2) { NGramBuffer bigram = getBigramBuffer(word1); NGramProbability bigramProbability = bigram.findNGram(word2); return ngramProbTable[1][bigramProbability.getProbabilityID()]; }
float logbgprob = ngramProbTable[1][bgProb.getProbabilityID()]; float smearTerm; NGramProbability bgProb = bigram.getNGramProbability(j); float logbgbackoff = ngramBackoffTable[2][bgProb.getBackoffID()]; double bgbackoff = logMath.logToLinear(logbgbackoff); int k = bigram.getWordID(j);
/** * Returns the NGramProbability of the nth follower. * * @param nthFollower which follower * @return the NGramProbability of the nth follower */ @Override public NGramProbability getNGramProbability(int nthFollower) { int nthPosition = 0, wordID = 0, probID = 0, backoffID = 0, firstNGram = 0; nthPosition = nthFollower * LargeNGramModel.BYTES_PER_NMAXGRAM * ((is32bits()) ? 4 : 2); setPosition(nthPosition); wordID = readBytesAsInt(); probID = readBytesAsInt(); return (new NGramProbability(nthFollower, wordID, probID, backoffID, firstNGram)); } }
/** * Returns the NGramProbability of the nth follower. * * @param nthFollower which follower * @return the NGramProbability of the nth follower */ public NGramProbability getNGramProbability(int nthFollower) { int nthPosition = 0, wordID = 0, probID = 0, backoffID = 0, firstNGram = 0; nthPosition = nthFollower * LargeNGramModel.BYTES_PER_NGRAM * ((is32bits) ? 4 : 2); setPosition(nthPosition); wordID = readBytesAsInt(); probID = readBytesAsInt(); backoffID = readBytesAsInt(); firstNGram = readBytesAsInt(); return (new NGramProbability(nthFollower, wordID, probID, backoffID, firstNGram)); } }