@Override protected void reduce(VarIntWritable key, Iterable<VectorOrPrefWritable> values, Context context) throws IOException, InterruptedException { List<Long> userIDs = Lists.newArrayList(); List<Float> prefValues = Lists.newArrayList(); Vector similarityMatrixColumn = null; for (VectorOrPrefWritable value : values) { if (value.getVector() == null) { // Then this is a user-pref value userIDs.add(value.getUserID()); prefValues.add(value.getValue()); } else { // Then this is the column vector if (similarityMatrixColumn != null) { throw new IllegalStateException("Found two similarity-matrix columns for item index " + key.get()); } similarityMatrixColumn = value.getVector(); } } if (similarityMatrixColumn == null) { return; } vectorAndPrefs.set(similarityMatrixColumn, userIDs, prefValues); context.write(key, vectorAndPrefs); }
@Override public void readFields(DataInput in) throws IOException { boolean hasVector = in.readBoolean(); if (hasVector) { VectorWritable writable = new VectorWritable(); writable.readFields(in); set(writable.get()); } else { long theUserID = Varint.readSignedVarLong(in); float theValue = in.readFloat(); set(theUserID, theValue); } }
public void map(IntWritable key, VectorWritable value, Context context) throws IOException, InterruptedException { context.write(key, new VectorOrPrefWritable(value.get())); } }
@Override public boolean matches(Object argument) { if (argument instanceof VectorOrPrefWritable) { Vector v = ((VectorOrPrefWritable) argument).getVector(); return MathHelper.consistsOf(v, elements); } return false; }
@Override protected void reduce(VarIntWritable key, Iterable<VectorOrPrefWritable> values, Context context) throws IOException, InterruptedException { List<Long> userIDs = new ArrayList<>(); List<Float> prefValues = new ArrayList<>(); Vector similarityMatrixColumn = null; for (VectorOrPrefWritable value : values) { if (value.getVector() == null) { // Then this is a user-pref value userIDs.add(value.getUserID()); prefValues.add(value.getValue()); } else { // Then this is the column vector if (similarityMatrixColumn != null) { throw new IllegalStateException("Found two similarity-matrix columns for item index " + key.get()); } similarityMatrixColumn = value.getVector(); } } if (similarityMatrixColumn == null) { return; } vectorAndPrefs.set(similarityMatrixColumn, userIDs, prefValues); context.write(key, vectorAndPrefs); }
@Override protected void map(IntWritable key, VectorWritable value, Context context) throws IOException, InterruptedException { Vector similarityMatrixRow = value.get(); /* remove self similarity */ similarityMatrixRow.set(key.get(), Double.NaN); index.set(key.get()); vectorOrPref.set(similarityMatrixRow); context.write(index, vectorOrPref); }
public void map(VarLongWritable key, VectorWritable value, Context context) throws IOException, InterruptedException { long userID = key.get(); Vector userVector = value.get(); Iterator<Vector.Element> it = userVector.iterateNonZero(); IntWritable itemIndexWritable = new IntWritable(); while (it.hasNext()) { Vector.Element e = it.next(); int itemIndex = e.index(); float preferenceValue = (float) e.get(); itemIndexWritable.set(itemIndex); context.write(itemIndexWritable, new VectorOrPrefWritable(userID, preferenceValue)); } } }
@Override protected void reduce(VarIntWritable key, Iterable<VectorOrPrefWritable> values, Context context) throws IOException, InterruptedException { List<Long> userIDs = Lists.newArrayList(); List<Float> prefValues = Lists.newArrayList(); Vector similarityMatrixColumn = null; for (VectorOrPrefWritable value : values) { if (value.getVector() == null) { // Then this is a user-pref value userIDs.add(value.getUserID()); prefValues.add(value.getValue()); } else { // Then this is the column vector if (similarityMatrixColumn != null) { throw new IllegalStateException("Found two similarity-matrix columns for item index " + key.get()); } similarityMatrixColumn = value.getVector(); } } if (similarityMatrixColumn == null) { return; } vectorAndPrefs.set(similarityMatrixColumn, userIDs, prefValues); context.write(key, vectorAndPrefs); }
@Override public void readFields(DataInput in) throws IOException { boolean hasVector = in.readBoolean(); if (hasVector) { VectorWritable writable = new VectorWritable(); writable.readFields(in); set(writable.get()); } else { long theUserID = Varint.readSignedVarLong(in); float theValue = in.readFloat(); set(theUserID, theValue); } }
/** * tests {@link ToVectorAndPrefReducer} */ @Test public void testToVectorAndPrefReducer() throws Exception { Reducer<VarIntWritable,VectorOrPrefWritable,VarIntWritable,VectorAndPrefsWritable>.Context context = EasyMock.createMock(Reducer.Context.class); context.write(EasyMock.eq(new VarIntWritable(1)), vectorAndPrefsWritableMatches(Arrays.asList(123L, 456L), Arrays.asList(1.0f, 2.0f), MathHelper.elem(3, 0.5), MathHelper.elem(7, 0.8))); EasyMock.replay(context); Vector similarityColumn = new RandomAccessSparseVector(Integer.MAX_VALUE, 100); similarityColumn.set(3, 0.5); similarityColumn.set(7, 0.8); VectorOrPrefWritable itemPref1 = new VectorOrPrefWritable(123L, 1.0f); VectorOrPrefWritable itemPref2 = new VectorOrPrefWritable(456L, 2.0f); VectorOrPrefWritable similarities = new VectorOrPrefWritable(similarityColumn); new ToVectorAndPrefReducer().reduce(new VarIntWritable(1), Arrays.asList(itemPref1, itemPref2, similarities), context); EasyMock.verify(context); }
@Override public void readFields(DataInput in) throws IOException { boolean hasVector = in.readBoolean(); if (hasVector) { VectorWritable writable = new VectorWritable(); writable.readFields(in); set(writable.get()); } else { long theUserID = Varint.readSignedVarLong(in); float theValue = in.readFloat(); set(theUserID, theValue); } }
/** * tests {@link ToVectorAndPrefReducer} in the error case that two similarity column vectors a supplied for the same * item (which should never happen) */ @Test public void testToVectorAndPrefReducerExceptionOn2Vectors() throws Exception { Reducer<VarIntWritable,VectorOrPrefWritable,VarIntWritable,VectorAndPrefsWritable>.Context context = EasyMock.createMock(Reducer.Context.class); EasyMock.replay(context); Vector similarityColumn1 = new RandomAccessSparseVector(Integer.MAX_VALUE, 100); Vector similarityColumn2 = new RandomAccessSparseVector(Integer.MAX_VALUE, 100); VectorOrPrefWritable similarities1 = new VectorOrPrefWritable(similarityColumn1); VectorOrPrefWritable similarities2 = new VectorOrPrefWritable(similarityColumn2); try { new ToVectorAndPrefReducer().reduce(new VarIntWritable(1), Arrays.asList(similarities1, similarities2), context); fail(); } catch (IllegalStateException e) { // good } EasyMock.verify(context); }
@Override protected void map(IntWritable key, VectorWritable value, Context context) throws IOException, InterruptedException { Vector similarityMatrixRow = value.get(); /* remove self similarity */ similarityMatrixRow.set(key.get(), Double.NaN); index.set(key.get()); vectorOrPref.set(similarityMatrixRow); context.write(index, vectorOrPref); }
@Override protected void map(IntWritable key, VectorWritable value, Context context) throws IOException, InterruptedException { Vector similarityMatrixRow = value.get(); /* remove self similarity */ similarityMatrixRow.set(key.get(), Double.NaN); index.set(key.get()); vectorOrPref.set(similarityMatrixRow); context.write(index, vectorOrPref); }
@Override protected void map(VarLongWritable key, VectorWritable value, Context context) throws IOException, InterruptedException { long userID = key.get(); log.info("UserID = {}", userID); if (usersToRecommendFor != null && !usersToRecommendFor.contains(userID)) { return; } Vector userVector = maybePruneUserVector(value.get()); for (Element e : userVector.nonZeroes()) { itemIndexWritable.set(e.index()); vectorOrPref.set(userID, (float) e.get()); context.write(itemIndexWritable, vectorOrPref); } }
@Override protected void map(VarLongWritable key, VectorWritable value, Context context) throws IOException, InterruptedException { long userID = key.get(); if (usersToRecommendFor != null && !usersToRecommendFor.contains(userID)) { return; } Vector userVector = maybePruneUserVector(value.get()); for (Element e : userVector.nonZeroes()) { itemIndexWritable.set(e.index()); vectorOrPref.set(userID, (float) e.get()); context.write(itemIndexWritable, vectorOrPref); } }
@Override protected void map(VarLongWritable key, VectorWritable value, Context context) throws IOException, InterruptedException { long userID = key.get(); if (usersToRecommendFor != null && !usersToRecommendFor.contains(userID)) { return; } Vector userVector = maybePruneUserVector(value.get()); for (Element e : userVector.nonZeroes()) { itemIndexWritable.set(e.index()); vectorOrPref.set(userID, (float) e.get()); context.write(itemIndexWritable, vectorOrPref); } }