public PartitionedFeatureVectors(int numPartitions, ExecutorService executor, ToIntBiFunction<String,float[]> partitioner) { Preconditions.checkArgument(numPartitions > 0); Objects.requireNonNull(executor); Objects.requireNonNull(partitioner); partitions = new FeatureVectorsPartition[numPartitions]; for (int i = 0; i < numPartitions; i++) { partitions[i] = new FeatureVectorsPartition(); } partitionMap = ObjectIntHashMap.newMap(); partitionMapLock = new AutoReadWriteLock(); this.partitioner = partitioner; this.executor = executor; }
@Override public void addAllRecentTo(Collection<String> allRecent) { try (AutoLock al = lock.autoReadLock()) { allRecent.addAll(recentIDs); } }
void setUserVector(String user, float[] vector) { Preconditions.checkArgument(vector.length == features); X.setVector(user, vector); try (AutoLock al = expectedUserIDsLock.autoWriteLock()) { expectedUserIDs.remove(user); } }
@Test public void testDefault() throws Exception { AutoReadWriteLock al = new AutoReadWriteLock(); assertNotNull(al.toString()); assertNotNull(al.readLock()); assertNotNull(al.writeLock()); }
/** * Like {@link #retainRecentAndUserIDs(Collection)} and {@link #retainRecentAndItemIDs(Collection)} * but affects the known-items data structure. * * @param users users that should be retained, which are coming in the new model updates * @param items items that should be retained, which are coming in the new model updates */ void retainRecentAndKnownItems(Collection<String> users, Collection<String> items) { // Keep all users in the new model, or, that have been added since last model MutableSet<String> recentUserIDs = UnifiedSet.newSet(); X.addAllRecentTo(recentUserIDs); try (AutoLock al = knownItemsLock.autoWriteLock()) { knownItems.keySet().removeIf(key -> !users.contains(key) && !recentUserIDs.contains(key)); } // This will be easier to quickly copy the whole (smallish) set rather than // deal with locks below MutableSet<String> allRecentKnownItems = UnifiedSet.newSet(); Y.addAllRecentTo(allRecentKnownItems); Predicate<String> notKeptOrRecent = value -> !items.contains(value) && !allRecentKnownItems.contains(value); try (AutoLock al = knownItemsLock.autoReadLock()) { knownItems.values().forEach(knownItemsForUser -> { synchronized (knownItemsForUser) { knownItemsForUser.removeIf(notKeptOrRecent); } }); } }
@Test public void testReadLock() { ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); AutoReadWriteLock al = new AutoReadWriteLock(lock); assertEquals(0, lock.getReadLockCount()); try (AutoLock al2 = al.autoReadLock()) { assertEquals(1, lock.getReadLockCount()); } assertEquals(0, lock.getReadLockCount()); }
@Test public void testWriteLock() { ReentrantReadWriteLock lock = new ReentrantReadWriteLock(); AutoReadWriteLock al = new AutoReadWriteLock(lock); assertFalse(lock.isWriteLocked()); try (AutoLock al2 = al.autoWriteLock()) { assertTrue(lock.isWriteLocked()); } assertFalse(lock.isWriteLocked()); }
/** * @param id ID to remove feature vector for */ void removeVector(String id) { try (AutoLock al = lock.autoWriteLock()) { vectors.remove(id); recentIDs.remove(id); } }
private MutableSet<String> doGetKnownItems(String user) { try (AutoLock al = knownItemsLock.autoReadLock()) { return knownItems.get(user); } }
public FeatureVectorsPartition() { vectors = UnifiedMap.newMap(); recentIDs = UnifiedSet.newSet(); lock = new AutoReadWriteLock(); }
@Override public void setVector(String id, float[] vector) { try (AutoLock al = lock.autoWriteLock()) { if (vectors.put(id, vector) == null) { // ID was actually new recentIDs.add(id); } } }
@Override public void removeAllIDsFrom(Collection<String> allIDs) { try (AutoLock al = lock.autoReadLock()) { allIDs.removeAll(vectors.keySet()); } }
/** * Creates an empty model. * * @param features number of features expected for user/item feature vectors * @param implicit whether model implements implicit feedback * @param logStrength whether input strengths are log transformed * @param epsilon eps in log transform log(1 + r/eps) */ ALSSpeedModel(int features, boolean implicit, boolean logStrength, double epsilon) { Preconditions.checkArgument(features > 0); int numPartitions = Runtime.getRuntime().availableProcessors(); X = new PartitionedFeatureVectors(numPartitions, executor); Y = new PartitionedFeatureVectors(numPartitions, executor); expectedUserIDs = UnifiedSet.newSet(); expectedUserIDsLock = new AutoReadWriteLock(); expectedItemIDs = UnifiedSet.newSet(); expectedItemIDsLock = new AutoReadWriteLock(); this.features = features; this.implicit = implicit; this.logStrength = logStrength; this.epsilon = epsilon; cachedXTXSolver = new SolverCache(executor, X); cachedYTYSolver = new SolverCache(executor, Y); }
@Override public void retainRecentAndIDs(Collection<String> newModelIDs) { try (AutoLock al = lock.autoWriteLock()) { vectors.keySet().removeIf(key -> !newModelIDs.contains(key) && !recentIDs.contains(key)); recentIDs.clear(); } }
/** * @param action function to apply to every ID/vector pair */ public void forEach(BiConsumer<String,float[]> action) { try (AutoLock al = lock.autoReadLock()) { vectors.forEach(action); } }
/** * Creates an empty model. * * @param features number of features expected for user/item feature vectors * @param implicit whether model implements implicit feedback * @param sampleRate consider only approximately this fraction of all items when making recommendations. * Candidates are chosen intelligently with locality sensitive hashing. * @param rescorerProvider optional instance of a {@link RescorerProvider} */ ALSServingModel(int features, boolean implicit, double sampleRate, RescorerProvider rescorerProvider) { Preconditions.checkArgument(features > 0); Preconditions.checkArgument(sampleRate > 0.0 && sampleRate <= 1.0); lsh = new LocalitySensitiveHash(sampleRate, features); X = new FeatureVectorsPartition(); Y = new PartitionedFeatureVectors( lsh.getNumPartitions(), executor, (String id, float[] vector) -> lsh.getIndexFor(vector)); knownItems = UnifiedMap.newMap(); knownItemsLock = new AutoReadWriteLock(); expectedUserIDs = UnifiedSet.newSet(); expectedUserIDsLock = new AutoReadWriteLock(); expectedItemIDs = UnifiedSet.newSet(); expectedItemIDsLock = new AutoReadWriteLock(); cachedYTYSolver = new SolverCache(executor, Y); this.features = features; this.implicit = implicit; this.rescorerProvider = rescorerProvider; }
void setItemVector(String item, float[] vector) { Preconditions.checkArgument(vector.length == features); Y.setVector(item, vector); try (AutoLock al = expectedItemIDsLock.autoWriteLock()) { expectedItemIDs.remove(item); } // Not clear if it's too inefficient to clear and recompute YtY solver every time any bit // of Y changes, but it's the most correct cachedYTYSolver.setDirty(); }
@Override public float[] getVector(String id) { try (AutoLock al = lock.autoReadLock()) { return vectors.get(id); } }
public PartitionedFeatureVectors(int numPartitions, ExecutorService executor, ToIntBiFunction<String,float[]> partitioner) { Preconditions.checkArgument(numPartitions > 0); Objects.requireNonNull(executor); Objects.requireNonNull(partitioner); partitions = new FeatureVectorsPartition[numPartitions]; for (int i = 0; i < numPartitions; i++) { partitions[i] = new FeatureVectorsPartition(); } partitionMap = ObjectIntHashMap.newMap(); partitionMapLock = new AutoReadWriteLock(); this.partitioner = partitioner; this.executor = executor; }
public void setUserVector(String user, float[] vector) { Preconditions.checkArgument(vector.length == features); X.setVector(user, vector); try (AutoLock al = expectedUserIDsLock.autoWriteLock()) { expectedUserIDs.remove(user); } cachedXTXSolver.setDirty(); }