@Override public void getFromOffset(final long offset, final int ngramOrder, @OutputParameter final PhraseTableValues outputVal) { if (offset >= valueIndexes[ngramOrder].size()) return; final long valueIndex = valueIndexes[ngramOrder].get(offset); if (valueIndex == EMPTY_VALUE_INDEX) return; if (outputVal instanceof FeaturePhraseTableValues && valueIndex >= 0) { final float[] fs = new float[numFeatures]; for (int i = 0; i < numFeatures; ++i) fs[i] = Float.intBitsToFloat((int) features[ngramOrder].get((int) (valueIndex + i))); ((FeaturePhraseTableValues) outputVal).features = fs; } if (outputVal instanceof TargetTranslationsValues && valueIndex < 0) { ((TargetTranslationsValues) outputVal).targetTranslationOffsets = readOffsets(targetTranslations[ngramOrder].get((int) (-valueIndex - 1))); ((TargetTranslationsValues) outputVal).targetTranslationOrders = readOrders(targetTranslations[ngramOrder].get((int) (-valueIndex - 1))); } }
public long getValueAndOffset(final long contextOffset, final int contextOrder, int word, @OutputParameter V currProbVal);
/** * @param contextOffset_ * @param contextOrder * @param word * @param logFailure * @return */ private long getOffsetForContextEncoding(final long contextOffset_, final int contextOrder, final int word, @OutputParameter final T outputVal) { if (word < 0) return -1; final int ngramOrder = contextOrder + 1; final long contextOffset = contextOffset_ >= 0 ? contextOffset_ : 0; final long key = combineToKey(word, contextOffset); final long offset = getOffsetHelpFromMap(ngramOrder, key); if (outputVal != null && offset >= 0) { values.getFromOffset(offset, ngramOrder, outputVal); } return offset; }
@Override public abstract float getLogProb(long contextOffset, int contextOrder, int word, @OutputParameter LmContextInfo outputContext);
@Override public void getFromOffset(final long offset, final int ngramOrder, @OutputParameter final KneserNeyCounts outputVal) { final boolean isHighestOrder = isHighestOrder(ngramOrder); final boolean isSecondHighestOrder = isSecondHighestOrder(ngramOrder); outputVal.tokenCounts = isHighestOrder ? tokenCounts.get(offset) : (isSecondHighestOrder ? getSafe(offset, prefixTokenCounts) : -1); outputVal.rightDotTypeCounts = (int) ((isHighestOrder || (offset >= rightDotTypeCounts[ngramOrder].size())) ? -1 : rightDotTypeCounts[ngramOrder] .get(offset)); outputVal.leftDotTypeCounts = (int) ((isHighestOrder || (offset >= leftDotTypeCounts[ngramOrder].size())) ? -1 : leftDotTypeCounts[ngramOrder] .get(offset)); outputVal.dotdotTypeCounts = (int) ((isHighestOrder || isSecondHighestOrder || (offset >= dotdotTypeCounts[ngramOrder].size())) ? -1 : dotdotTypeCounts[ngramOrder].get(offset)); outputVal.isOneCount = oneCountOffsets[ngramOrder].containsKey(offset); outputVal.isTwoCount = twoCountOffsets[ngramOrder].containsKey(offset); outputVal.isInternal = true; }
abstract protected void getFromRank(final int rank, @OutputParameter V outputVal);
@Override public float getCached(final long contextOffset, final int contextOrder, final int word, final int hash, @OutputParameter final LmContextInfo outputPrefix) { final long[] array = !threadSafe ? threadUnsafeArray : threadSafeArray.get(); final int cachedWordHere = getWord(hash, array); if (word >= 0 && word == cachedWordHere && getLong(hash, CONTEXT_OFFSET, array) == combine(contextOrder, contextOffset)) { final float f = getVal(hash, array); if (outputPrefix == null) return f; final long outputOrderAndOffset = getLong(hash, OUTPUT_CONTEXT_OFFSET, array); if (outputOrderAndOffset >= 0) { outputPrefix.order = orderOf(outputOrderAndOffset); outputPrefix.offset = offsetOf(outputOrderAndOffset); return f; } } return Float.NaN; }
/** * @param contextOffset_ * @param contextOrder * @param word * @return */ private long getOffsetForContextEncoding(final long contextOffset_, final int contextOrder, final int word, @OutputParameter final T outputVal) { final int ngramOrder = contextOrder + 1; final long offset = getOffsetHelp(contextOffset_, word, ngramOrder); if (offset >= 0 && outputVal != null) { values.getFromOffset(offset, ngramOrder, outputVal); } return offset; }
public void putCached(long contextOffset, int contextOrder, int word, float prob, int hash, @OutputParameter LmContextInfo outputPrefix);
@Override public float getCached(final long contextOffset, final int contextOrder, final int word, final int hash, @OutputParameter final LmContextInfo outputPrefix) { final float f = getVal(hash); final long outputContextOffset = getOutputContextOffset(hash); if (!Float.isNaN(f) && (outputPrefix == null || outputContextOffset >= 0)) { final int cachedWordHere = getWord(hash); if (cachedWordHere != -1 && equals(contextOffset, contextOrder, word, getContextOffset(hash), cachedWordHere, getContextOrder(hash))) { if (outputPrefix != null) { outputPrefix.order = getOutputContextOrder(hash); outputPrefix.offset = outputContextOffset; } return f; } } return Float.NaN; }
@Override public abstract float getLogProb(long contextOffset, int contextOrder, int word, @OutputParameter LmContextInfo outputContext);
@Override public long getValueAndOffset(final long contextOffset, final int contextOrder, final int word, @OutputParameter final T outputVal) { return getOffsetForContextEncoding(contextOffset, contextOrder, word, outputVal); }
@Override protected void getFromRank(final long rank, @OutputParameter final LongRef outputVal) { outputVal.value = countsForRank[(int) rank]; }
/** * Gets the value living at a particular offset. * * @param offset * @param ngramOrder * @return */ public void getFromOffset(long offset, int ngramOrder, @OutputParameter V outputVal);
@Override public final void decompress(final BitStream bits, final int ngramOrder, final boolean justConsume, @OutputParameter final ProbBackoffPair outputVal) { final long probRank = valueCoder.decompress(bits); final long backoffRank = (ngramOrder < numNgramsForEachOrder.length - 1) ? valueCoder.decompress(bits) : -1; if (justConsume) return; if (outputVal != null) { outputVal.prob = probsForRank[(int) probRank]; outputVal.backoff = (ngramOrder < numNgramsForEachOrder.length - 1) ? backoffsForRank[(int) backoffRank] : 0; } }
/** * Reads and decompresses from the bit stream bits. * * @param bits * @param ngramOrder * @param justConsume * If true, nothing is returned, and the function simply consumes * the appropriate number of bits from the BitStream. * * @return */ public void decompress(BitStream bits, int ngramOrder, boolean justConsume, @OutputParameter V outputVal);
@Override protected void getFromRank(final long rank, @OutputParameter final ProbBackoffPair outputVal) { outputVal.prob = getFromRank(rank, false); outputVal.backoff = getFromRank(rank, true); }
@Override public void getFromOffset(final long index, final int ngramOrder, @OutputParameter final ProbBackoffPair outputVal) { final long rank = getRank(ngramOrder, index); getFromRank(rank, outputVal); }
@Override public void putCached(final long contextOffset, final int contextOrder, final int word, final float score, final int hash, @OutputParameter final LmContextInfo outputPrefix) { setWord(hash, word); setVal(hash, score); setContextOffset(hash, contextOffset); setContextOrder(hash, contextOrder); setOutputContextOrder(hash, outputPrefix == null ? -1 : outputPrefix.order); setOutputContextOffset(hash, outputPrefix == null ? -1 : outputPrefix.offset); }
@Override public long getValueAndOffset(final long contextOffset, final int contextNgramOrder, final int word, @OutputParameter final T outputVal) { final long hash = combineToKey(word, contextOffset); final int ngramOrder = contextNgramOrder + 1; final LongArray compressedKeys = (maps[ngramOrder]).compressedKeys; final long currIndex = decompressSearch(compressedKeys, hash, ngramOrder, outputVal); return currIndex; }