/** * Convert a map entry to a basic result. * @param entry The map entry. * @return The basic result. */ public static BasicResult fromEntry(Map.Entry<Long,Double> entry) { if (entry instanceof Long2DoubleMap.Entry) { Long2DoubleMap.Entry e = (Long2DoubleMap.Entry) entry; return create(e.getLongKey(), e.getDoubleValue()); } else { return create(entry.getKey(), entry.getValue()); } }
if (nusers != 0) { double currentDev = model.getDeviation(predicteeItem, currentItem); total += currentDev + e.getDoubleValue(); nitems++;
@Nonnull @Override public Map<String, Object> measureUser(Recommender recommender, TestUser testUser) { RatingPredictor predictor = recommender.getRatingPredictor(); if (predictor == null) { logger.debug("recommender cannot predict ratings"); return Collections.emptyMap(); } Long2DoubleMap ratings = testUser.getTestRatings(); ResultMap results = predictor.predictWithDetails(testUser.getUserId(), ratings.keySet()); // Measure the user results Map<String,Object> row = new HashMap<>(); for (MetricContext<?> mc: predictMetricContexts) { row.putAll(mc.measureUser(testUser, results).getValues()); } // Write all attempted predictions for (Long2DoubleMap.Entry e: ratings.long2DoubleEntrySet()) { Result pred = results.get(e.getLongKey()); try { if (writer != null) { writer.writeRow(testUser.getUserId(), e.getLongKey(), e.getDoubleValue(), pred != null ? pred.getScore() : null); } } catch (IOException ex) { throw new EvaluationException("error writing prediction row", ex); } } return row; }
for (Long2DoubleMap.Entry e: sums.long2DoubleEntrySet()) { long user = e.getLongKey(); double usum = e.getDoubleValue(); double ucount = counts.get(user); usum += damping * mean;
/** * Compute the dot product of two maps. This method assumes any value missing in one map is 0, so it is the dot * product of the values of common keys. * @param v1 The first vector. * @param v2 The second vector. * @return The sum of the products of corresponding values in the two vectors. */ public static double dotProduct(Long2DoubleMap v1, Long2DoubleMap v2) { if (v1.size() > v2.size()) { // compute dot product the other way around for speed return dotProduct(v2, v1); } if (v1 instanceof Long2DoubleSortedArrayMap && v2 instanceof Long2DoubleSortedArrayMap) { return dotProduct((Long2DoubleSortedArrayMap) v1, (Long2DoubleSortedArrayMap) v2); } else { double result = 0; Long2DoubleFunction v2d = adaptDefaultValue(v2, 0.0); Iterator<Long2DoubleMap.Entry> iter = fastEntryIterator(v1); while (iter.hasNext()) { Long2DoubleMap.Entry e = iter.next(); long k = e.getLongKey(); result += e.getDoubleValue() * v2d.get(k); // since default is 0 } return result; } }
/** * Score a single item into an accumulator. * @param scores The reference scores. * @param item The item to score. * @param accum The accumulator. */ protected void scoreItem(Long2DoubleMap scores, long item, ItemItemScoreAccumulator accum) { Long2DoubleMap allNeighbors = model.getNeighbors(item); Long2DoubleAccumulator acc; if (neighborhoodSize > 0) { // FIXME Abstract accumulator selection logic acc = new TopNLong2DoubleAccumulator(neighborhoodSize); } else { acc = new UnlimitedLong2DoubleAccumulator(); } for (Long2DoubleMap.Entry nbr: allNeighbors.long2DoubleEntrySet()) { if (scores.containsKey(nbr.getLongKey())) { acc.put(nbr.getLongKey(), nbr.getDoubleValue()); } } Long2DoubleMap neighborhood = acc.finishMap(); scorer.score(item, neighborhood, scores, accum); } }
protected void scoreItem(Long2DoubleMap userData, long item, ItemItemScoreAccumulator accum) { // find the usable neighbors Long2DoubleSortedArrayMap allNeighbors = Long2DoubleSortedArrayMap.create(model.getNeighbors(item)); Long2DoubleMap neighborhood = allNeighbors.subMap(userData.keySet()); if (neighborhoodSize > 0) { if (logger.isTraceEnabled()) { logger.trace("truncating {} neighbors to {}", neighborhood.size(), neighborhoodSize); } Long2DoubleAccumulator acc = new TopNLong2DoubleAccumulator(neighborhoodSize); for (Long2DoubleMap.Entry e: neighborhood.long2DoubleEntrySet()) { acc.put(e.getLongKey(), e.getDoubleValue()); } neighborhood = acc.finishMap(); } assert neighborhoodSize <= 0 || neighborhood.size() <= neighborhoodSize; if (neighborhood.size() < minNeighbors) { return; } if (logger.isTraceEnabled()) { logger.trace("scoring item {} with {} of {} neighbors", item, neighborhood.size(), allNeighbors.size()); } scorer.score(item, neighborhood, userData, accum); } }
/** * Implement recommendation by calling {@link ItemScorer#score(long, Collection)} and sorting * the results by score. This method uses {@link #getDefaultExcludes(long)} to get the default * exclude set for the user, if none is provided. */ @Override protected List<Long> recommend(long user, int n, LongSet candidates, LongSet exclude) { candidates = getEffectiveCandidates(user, candidates, exclude); logger.debug("Computing {} recommendations for user {} from {} candidates", n, user, candidates.size()); Map<Long, Double> scores = scorer.score(user, candidates); Long2DoubleAccumulator accum; if (n >= 0) { accum = new TopNLong2DoubleAccumulator(n); } else { accum = new UnlimitedLong2DoubleAccumulator(); } Long2DoubleMap map = LongUtils.asLong2DoubleMap(scores); for (Long2DoubleMap.Entry e: Vectors.fastEntries(map)) { accum.put(e.getLongKey(), e.getDoubleValue()); } return accum.finishList(); }
@Test public void testUnitVector() { for (Map<Long,Double> map: someMaps(longs(), doubles(-100, 100))) { if (map.isEmpty()) { continue; } Long2DoubleMap vec = LongUtils.frozenMap(map); double norm = Vectors.euclideanNorm(vec); Long2DoubleMap unit = Vectors.unitVector(vec); assertThat(unit.size(), equalTo(vec.size())); assertThat(unit.keySet(), equalTo(vec.keySet())); assertThat(Vectors.euclideanNorm(unit), closeTo(1.0, 1.0e-6)); Long2DoubleMaps.fastForEach(unit, e -> { assertThat(e.getDoubleValue() * norm, closeTo(vec.get(e.getLongKey()), 1.0e-6)); }); } } }
@Override public BinaryOperator<Long2ObjectMap<Long2DoubleAccumulator>> combiner() { return (am1, am2) -> { for (Long2ObjectMap.Entry<Long2DoubleAccumulator> e: Long2ObjectMaps.fastIterable(am2)) { long item = e.getLongKey(); Long2DoubleAccumulator a2 = e.getValue(); Long2DoubleAccumulator a1 = am1.get(item); if (a1 == null) { am1.put(item, a2); } else { for (Long2DoubleMap.Entry ae: Long2DoubleMaps.fastIterable(a2.finishMap())) { a1.put(ae.getLongKey(), ae.getDoubleValue()); } } } return am1; }; }
@Override public UserItemBiasModel get() { double intercept = itemBiases.getIntercept(); Long2DoubleMap itemOff = itemBiases.getItemBiases(); Long2DoubleMap map = new Long2DoubleOpenHashMap(); try (ObjectStream<IdBox<Long2DoubleMap>> stream = dao.streamUsers()) { for (IdBox<Long2DoubleMap> user : stream) { Long2DoubleMap uvec = user.getValue(); double usum = 0; for (Long2DoubleMap.Entry e: uvec.long2DoubleEntrySet()) { double off = itemOff.get(e.getLongKey()); usum += e.getDoubleValue() - intercept - off; } map.put(user.getId(), usum / (uvec.size() + damping)); } } return new UserItemBiasModel(intercept, map, itemOff); } }
@Test public void testSimpleTruncate() { long[] keys = {1, 2, 3, 4}; double[] values = {1.0, 2.0, 3.0, 4.0}; Long2DoubleSortedArrayMap v = Long2DoubleSortedArrayMap.wrapUnsorted(keys, values); VectorTruncator truncator = new NoOpTruncator(); Long2DoubleMap v2 = truncator.truncate(v); long i = 1; for (Long2DoubleMap.Entry e: v2.long2DoubleEntrySet()) { assertThat(e.getLongKey(), equalTo(i)); assertThat(e.getDoubleValue(), closeTo(i, EPSILON)); i++; } assertThat(i, equalTo(5L)); } }
@Test public void testSimpleTruncate() { long[] keys = {1, 2, 3, 4, 5}; double[] values = {1.0, 2.0, 3.0, 4.0, 5.0}; Long2DoubleSortedArrayMap v = Long2DoubleSortedArrayMap.wrapUnsorted(keys, values); VectorTruncator truncator = new TopNTruncator(3, null); Long2DoubleMap v2 = truncator.truncate(v); long i = 3; for (Long2DoubleMap.Entry e: v2.long2DoubleEntrySet()) { assertThat(e.getLongKey(), equalTo(i)); assertThat(e.getDoubleValue(), closeTo(i, EPSILON)); i++; } assertThat(i, equalTo(6L)); } }
@Test public void testTruncate() { long[] keys = {1, 2, 3, 4}; double[] values = {1.0, 2.0, 3.0, 4.0}; Long2DoubleSortedArrayMap v = Long2DoubleSortedArrayMap.wrapUnsorted(keys, values); VectorTruncator truncator = new ThresholdTruncator(new RealThreshold(3.5)); Long2DoubleMap v2 = truncator.truncate(v); int numSeen = 0; for (Long2DoubleMap.Entry e: v2.long2DoubleEntrySet()) { assertThat(e.getLongKey(), equalTo(4L)); assertThat(e.getDoubleValue(), closeTo(4.0, EPSILON)); numSeen++; } assertThat(numSeen, equalTo(1)); } }
@Override public double getUserBias(long user) { Long2DoubleMap vec = dao.userRatingVector(user); if (vec.isEmpty()) { return 0; } else { double sum = 0; double mean = getIntercept(); for (Long2DoubleMap.Entry e: vec.long2DoubleEntrySet()) { sum += e.getDoubleValue() - mean - getItemBias(e.getLongKey()); } return sum / vec.size(); } }
@Override public BiConsumer<Long2ObjectMap<Long2DoubleAccumulator>, IdBox<Long2DoubleMap>> accumulator() { return (acc, row) -> { long i1 = row.getId(); for (Long2DoubleMap.Entry e: Long2DoubleMaps.fastIterable(row.getValue())) { long i2 = e.getLongKey(); double sim = e.getDoubleValue(); acc.computeIfAbsent(i1, i -> newAccumulator()) .put(i2, sim); acc.computeIfAbsent(i2, i -> newAccumulator()) .put(i1, sim); } }; }
@Override public Long2DoubleMap truncate(Long2DoubleMap v) { Long2DoubleMap res = new Long2DoubleOpenHashMap(v.size()); for (Long2DoubleMap.Entry e: Vectors.fastEntries(v)) { if (threshold.retain(e.getDoubleValue())) { res.put(e.getLongKey(), e.getDoubleValue()); } } return res; } }
/** * Add all items from a map to the accumulator. * @param map The map. */ default void putAll(Long2DoubleMap map) { for (Long2DoubleMap.Entry e: Long2DoubleMaps.fastIterable(map)) { put(e.getLongKey(), e.getDoubleValue()); } }
@Override public Long2DoubleMap truncate(Long2DoubleMap v) { TopNLong2DoubleAccumulator accumulator = new TopNLong2DoubleAccumulator(n); for (Long2DoubleMap.Entry e : Vectors.fastEntries(v)) { double x = e.getDoubleValue(); if (threshold == null || threshold.retain(x)) { accumulator.put(e.getLongKey(), x); } } return accumulator.finishMap(); } }
@Override public double similarity(Long2DoubleMap vec1, Long2DoubleMap vec2) { MutualInformationAccumulator accum = new MutualInformationAccumulator(quantizer.getCount()); for (Long2DoubleMap.Entry e: vec1.long2DoubleEntrySet()) { long k = e.getLongKey(); if (vec2.containsKey(k)) { accum.count(quantizer.index(e.getDoubleValue()), quantizer.index(vec2.get(k))); } } return accum.getMutualInformation(); }