public WeightedVectorWritable(double weight, Vector vector) { this.vectorWritable.set(vector); this.weight = weight; }
@Override protected void reduce(IntWritable row, Iterable<VectorWritable> vectors, Context ctx) throws IOException, InterruptedException { merged.setWritesLaxPrecision(true); merged.set(VectorWritable.mergeToVector(vectors.iterator())); ctx.write(row, merged); } }
@Override protected void reduce(WritableComparable<?> key, Iterable<VectorWritable> values, Context ctx) throws IOException, InterruptedException { result.set(Vectors.sum(values.iterator())); ctx.write(key, result); } }
@Override protected void reduce(WritableComparable<?> key, Iterable<VectorWritable> values, Context ctx) throws IOException, InterruptedException { result.set(Vectors.sum(values.iterator())); ctx.write(key, result); } }
@Override public void map(IntWritable key, WeightedVectorWritable val, Context context) throws IOException, InterruptedException { // by pivoting on the cluster mapping value, we can make sure that each unique cluster goes to it's own reducer, // since they are numbered from 0 to k-1, where k is the number of clusters outputVector.set(val.getVector()); context.write(new IntWritable(newClusterMappings.get(key.get())), outputVector); } }
@Override public void map(IntWritable key, WeightedVectorWritable val, Context context) throws IOException, InterruptedException { // by pivoting on the cluster mapping value, we can make sure that each unique cluster goes to it's own reducer, // since they are numbered from 0 to k-1, where k is the number of clusters outputVector.set(val.getVector()); context.write(new IntWritable(newClusterMappings.get(key.get())), outputVector); } }
@Override protected void reduce(WritableComparable<?> key, Iterable<VectorWritable> values, Context ctx) throws IOException, InterruptedException { Vector sum = Vectors.sum(values.iterator()); result.set(new SequentialAccessSparseVector(sum)); ctx.write(key, result); } }
@Override protected void setup(Context context) throws IOException, InterruptedException { int k = context.getConfiguration().getInt(PROP_K, -1); int p = context.getConfiguration().getInt(PROP_P, -1); Validate.isTrue(k > 0, "invalid k parameter"); Validate.isTrue(p > 0, "invalid p parameter"); accum.set(acc = new DenseVector(k + p)); }
@Override protected void map(IntWritable userOrItemID, VectorWritable ratingsWritable, Context ctx) throws IOException, InterruptedException { OpenIntObjectHashMap<Vector> uOrM = getSharedInstance(); uiOrmj.set(ALS.solveExplicit(ratingsWritable, uOrM, lambda, numFeatures)); ctx.write(userOrItemID, uiOrmj); }
@Override public void reduce(WritableComparable<?> key, Iterable<VectorWritable> vectors, Context ctx) throws IOException, InterruptedException { Vector merged = VectorWritable.merge(vectors.iterator()).get(); result.set(new SequentialAccessSparseVector(merged)); ctx.write(key, result); } }
@SuppressWarnings("unchecked") private void outputQRow(Writable key, Vector qRow, Vector aRow) throws IOException { if (nv && (aRow instanceof NamedVector)) { qRowValue.set(new NamedVector(qRow, ((NamedVector) aRow).getName())); } else { qRowValue.set(qRow); } outputs.getCollector(OUTPUT_Q, null).collect(key, qRowValue); } }
@SuppressWarnings("unchecked") private void outputQRow(Writable key, Vector qRow, Vector aRow) throws IOException { if (nv && (aRow instanceof NamedVector)) { qRowValue.set(new NamedVector(qRow, ((NamedVector) aRow).getName())); } else { qRowValue.set(qRow); } outputs.getCollector(OUTPUT_Q, null).collect(key, qRowValue); } }
@Override public void reduce(WritableComparable<?> key, Iterable<VectorWritable> vectors, Context ctx) throws IOException, InterruptedException { Vector merged = VectorWritable.merge(vectors.iterator()).get(); result.set(new SequentialAccessSparseVector(merged)); ctx.write(key, result); } }
@SuppressWarnings("unchecked") private void outputQRow(Writable key, Vector qRow, Vector aRow) throws IOException { if (nv && (aRow instanceof NamedVector)) { qRowValue.set(new NamedVector(qRow, ((NamedVector) aRow).getName())); } else { qRowValue.set(qRow); } outputs.getCollector(OUTPUT_Q, null).collect(key, qRowValue); } }
@Override protected void map(IntWritable userOrItemID, VectorWritable ratingsWritable, Context ctx) throws IOException, InterruptedException { ImplicitFeedbackAlternatingLeastSquaresSolver solver = getSharedInstance(); uiOrmj.set(solver.solve(ratingsWritable.get())); ctx.write(userOrItemID, uiOrmj); }
@Override public void reduce(WritableComparable<?> key, Iterable<VectorWritable> vectors, Context ctx) throws IOException, InterruptedException { Vector merged = VectorWritable.merge(vectors.iterator()).get(); result.set(new SequentialAccessSparseVector(merged)); ctx.write(key, result); ctx.getCounter(Stats.NUM_USERS).increment(1); } }
@Override public void reduce(WritableComparable<?> key, Iterable<VectorWritable> vectors, Context ctx) throws IOException, InterruptedException { Vector merged = VectorWritable.merge(vectors.iterator()).get(); result.set(new SequentialAccessSparseVector(merged)); ctx.write(key, result); ctx.getCounter(Stats.NUM_USERS).increment(1); } }
@Override public void write(DataOutput out) throws IOException { Varint.writeUnsignedVarInt(numRows, out); VectorWritable vw = new VectorWritable(); for (int i = 0; i < numRows; i++) { Varint.writeUnsignedVarInt(rowIndices[i], out); vw.set(rows[i]); vw.write(out); } }
public static void write(Path outputDir, Configuration conf, Iterable<MatrixSlice> matrix) throws IOException { FileSystem fs = outputDir.getFileSystem(conf); SequenceFile.Writer writer = SequenceFile.createWriter(fs, conf, outputDir, IntWritable.class, VectorWritable.class); IntWritable topic = new IntWritable(); VectorWritable vector = new VectorWritable(); for (MatrixSlice slice : matrix) { topic.set(slice.index()); vector.set(slice.vector()); writer.append(topic, vector); } writer.close(); }
@Override public void write(DataOutput out) throws IOException { Varint.writeUnsignedVarInt(numRows, out); VectorWritable vw = new VectorWritable(); for (int i = 0; i < numRows; i++) { Varint.writeUnsignedVarInt(rowIndices[i], out); vw.set(rows[i]); vw.write(out); } }