buildUnigramIDMap(dictionary); else buildUnigramIDMap(); loadedBigramBuffers = new NGramBuffer[unigrams.length]; try { System.out.println("... Reading ..."); readSmearInfo("smear.dat"); System.out.println("... Done "); } catch (IOException e) { System.out.println("... " + e); System.out.println("... Calculating"); buildSmearInfo(); System.out.println("... Writing");
@Override public void onUtteranceEnd() { clearCache(); if (logFile != null) { logFile.println("<END_UTT>"); logFile.flush(); } }
private Float getNGramProbability(WordSequence wordSequence) { int numberWords = wordSequence.size(); Word firstWord = wordSequence.getWord(0); if (loader.getNumberNGrams(numberWords) == 0 || !hasUnigram(firstWord)) return getNGramProbability(wordSequence.getNewest()); if (numberWords < 2) { return getUnigramProbability(wordSequence); } NGramProbability nGProbability = findNGram(wordSequence); if (nGProbability != null) { return ngramProbTable[numberWords - 1][nGProbability .getProbabilityID()]; } if (numberWords == 2) { UnigramProbability unigramProb = getUnigram(firstWord); UnigramProbability unigramProb1 = getUnigram(wordSequence .getWord(1)); return unigramProb.getLogBackoff() + unigramProb1.getLogProbability(); } NGramProbability nMinus1Gram = findNGram(wordSequence.getOldest()); if (nMinus1Gram != null) { return ngramBackoffTable[numberWords - 1][nMinus1Gram .getBackoffID()] + getProbability(wordSequence.getNewest()); } return getProbability(wordSequence.getNewest()); }
NGramBuffer bigram = getBigramBuffer(i); Float smearTerm = getSmearTerm(i, k); out.writeInt(k); out.writeFloat(smearTerm.floatValue());
+ loadedBigramBuffers.length); int numBigrams = in.readInt(); NGramBuffer bigram = getBigramBuffer(i); putSmearTerm(i, k, in.readFloat());
int firstWordID = getWordID(ws.getWord(0)); int firstCurrentNGramEntry = 0; int numberNGrams = 0; numberNGrams = getNumberBigramFollowers(firstWordID) + 1; int lastWordId = getWordID(ws.getWord(ws.size() - 1)); nMinus1Buffer = getNGramBuffer(ws.getOldest()); int index = nMinus1Buffer.findNGramIndex(lastWordId); firstCurrentNGramEntry = getFirstNGramEntry( nMinus1Buffer.getNGramProbability(index), firstNMinus1GramEntry, orderBuffer); int firstNextNGramEntry = getFirstNGramEntry( nMinus1Buffer.getNGramProbability(index + 1), firstNMinus1GramEntry, orderBuffer); loader.getBigEndian(), is32bits(), orderBuffer, firstCurrentNGramEntry); } else { currentBuffer = new NGramBuffer(buffer, numberNGrams, loader.getBigEndian(), is32bits(), orderBuffer, firstCurrentNGramEntry);
NGramBuffer bigram = getBigramBuffer(i); System.out.println("Processed " + i + " of " + loadedBigramBuffers.length); NGramBuffer bigram = getBigramBuffer(i); double bgbackoff = logMath.logToLinear(logbgbackoff); int k = bigram.getWordID(j); NGramBuffer trigram = loadTrigramBuffer(i, k); .getProbabilityID(l)]; double tgprob = logMath.logToLinear(logtgprob); float logbgprob = getBigramProb(k, m); double bgprob = logMath.logToLinear(logbgprob); float logugprob = unigrams[m].getLogProbability(); putSmearTerm(i, k, smearTerm);
/** * Finds or loads the NGram probability of the given NGram. * * @param wordSequence * the NGram to load * @return a NGramProbability of the given NGram */ private NGramProbability findNGram(WordSequence wordSequence) { int numberWords = wordSequence.size(); NGramProbability nGram = null; WordSequence oldest = wordSequence.getOldest(); NGramBuffer nGramBuffer = loadedNGramBuffers[numberWords - 1] .get(oldest); if (nGramBuffer == null) { nGramBuffer = getNGramBuffer(oldest); if (nGramBuffer != null) loadedNGramBuffers[numberWords - 1].put(oldest, nGramBuffer); } if (nGramBuffer != null) { int nthWordID = getWordID(wordSequence.getWord(numberWords - 1)); nGram = nGramBuffer.findNGram(nthWordID); } return nGram; }
/** * Retrieves the bigram probability for the two given words * * @param word1 * the first word of the bigram * @param word2 * the second word of the bigram * @return the log probability */ private float getBigramProb(int word1, int word2) { NGramBuffer bigram = getBigramBuffer(word1); NGramProbability bigramProbability = bigram.findNGram(word2); return ngramProbTable[1][bigramProbability.getProbabilityID()]; }
probability = getNGramProbability(wordSequence);
int firstWordID = getWordID(ws.getWord(0)); int firstCurrentNGramEntry = 0; int numberNGrams = 0; numberNGrams = getNumberBigramFollowers(firstWordID) + 1; int lastWordId = getWordID(ws.getWord(ws.size() - 1)); nMinus1Buffer = getNGramBuffer(ws.getOldest()); int index = nMinus1Buffer.findNGramIndex(lastWordId); firstCurrentNGramEntry = getFirstNGramEntry( nMinus1Buffer.getNGramProbability(index), firstNMinus1GramEntry, orderBuffer); int firstNextNGramEntry = getFirstNGramEntry( nMinus1Buffer.getNGramProbability(index + 1), firstNMinus1GramEntry, orderBuffer); loader.getBigEndian(), is32bits(), orderBuffer, firstCurrentNGramEntry); } else { currentBuffer = new NGramBuffer(buffer, numberNGrams, loader.getBigEndian(), is32bits(), orderBuffer, firstCurrentNGramEntry);
NGramBuffer bigram = getBigramBuffer(i); System.out.println("Processed " + i + " of " + loadedBigramBuffers.length); NGramBuffer bigram = getBigramBuffer(i); double bgbackoff = logMath.logToLinear(logbgbackoff); int k = bigram.getWordID(j); NGramBuffer trigram = loadTrigramBuffer(i, k); .getProbabilityID(l)]; double tgprob = logMath.logToLinear(logtgprob); float logbgprob = getBigramProb(k, m); double bgprob = logMath.logToLinear(logbgprob); float logugprob = unigrams[m].getLogProbability(); putSmearTerm(i, k, smearTerm);
/** * Finds or loads the NGram probability of the given NGram. * * @param wordSequence * the NGram to load * @return a NGramProbability of the given NGram */ private NGramProbability findNGram(WordSequence wordSequence) { int numberWords = wordSequence.size(); NGramProbability nGram = null; WordSequence oldest = wordSequence.getOldest(); NGramBuffer nGramBuffer = loadedNGramBuffers[numberWords - 1] .get(oldest); if (nGramBuffer == null) { nGramBuffer = getNGramBuffer(oldest); if (nGramBuffer != null) loadedNGramBuffers[numberWords - 1].put(oldest, nGramBuffer); } if (nGramBuffer != null) { int nthWordID = getWordID(wordSequence.getWord(numberWords - 1)); nGram = nGramBuffer.findNGram(nthWordID); } return nGram; }
NGramBuffer bigram = getBigramBuffer(i); Float smearTerm = getSmearTerm(i, k); out.writeInt(k); out.writeFloat(smearTerm.floatValue());
+ loadedBigramBuffers.length); int numBigrams = in.readInt(); NGramBuffer bigram = getBigramBuffer(i); putSmearTerm(i, k, in.readFloat());
/** * Retrieves the bigram probability for the two given words * * @param word1 * the first word of the bigram * @param word2 * the second word of the bigram * @return the log probability */ private float getBigramProb(int word1, int word2) { NGramBuffer bigram = getBigramBuffer(word1); NGramProbability bigramProbability = bigram.findNGram(word2); return ngramProbTable[1][bigramProbability.getProbabilityID()]; }
probability = getNGramProbability(wordSequence);
private Float getNGramProbability(WordSequence wordSequence) { int numberWords = wordSequence.size(); Word firstWord = wordSequence.getWord(0); if (loader.getNumberNGrams(numberWords) == 0 || !hasUnigram(firstWord)) return getNGramProbability(wordSequence.getNewest()); if (numberWords < 2) { return getUnigramProbability(wordSequence); } NGramProbability nGProbability = findNGram(wordSequence); if (nGProbability != null) { return ngramProbTable[numberWords - 1][nGProbability .getProbabilityID()]; } if (numberWords == 2) { UnigramProbability unigramProb = getUnigram(firstWord); UnigramProbability unigramProb1 = getUnigram(wordSequence .getWord(1)); return unigramProb.getLogBackoff() + unigramProb1.getLogProbability(); } NGramProbability nMinus1Gram = findNGram(wordSequence.getOldest()); if (nMinus1Gram != null) { return ngramBackoffTable[numberWords - 1][nMinus1Gram .getBackoffID()] + getProbability(wordSequence.getNewest()); } return getProbability(wordSequence.getNewest()); }
buildUnigramIDMap(dictionary); else buildUnigramIDMap(); loadedBigramBuffers = new NGramBuffer[unigrams.length]; try { System.out.println("... Reading ..."); readSmearInfo("smear.dat"); System.out.println("... Done "); } catch (IOException e) { System.out.println("... " + e); System.out.println("... Calculating"); buildSmearInfo(); System.out.println("... Writing");
@Override public void onUtteranceEnd() { clearCache(); if (logFile != null) { logFile.println("<END_UTT>"); logFile.flush(); } }