/** * converts a datamap's doc data into a vector object to be used in the creation of a LabeledPoint object * @param docdata * @param numfeatures * @return */ private static Vector getVector(Map<Integer,FeatureData> docdata, int numfeatures) { List<Integer> keys = new ArrayList<Integer>(docdata.keySet().size()); List<Double> values = new ArrayList<Double>(docdata.keySet().size()); for (Integer i : docdata.keySet()){ keys.add(i); values.add(docdata.get(i).getValue()); } return Vectors.sparse(numfeatures, Ints.toArray(keys), Doubles.toArray(values)); } }
public void call(Row event) { SparseVector sp = (SparseVector) event.getAs("normFeature"); double prediction = linModel.predict(Vectors.sparse(sp.size(), sp.indices(), sp.values())); System.out.println((String) event.getAs("document")); System.out.println("Prediction: " + prediction); } });
Vector features = Vectors.sparse(featureMap.size(), x); double label = classifier.predict(features);
Vector features = MLUtils.appendBias(Vectors.sparse(featureMap.size(), x)); int numLabels = tags.length; double[] score = new double[numLabels];
@Test public void sparseArrayConstruction() { @SuppressWarnings("unchecked") Vector v = Vectors.sparse(3, Arrays.asList( new Tuple2<>(0, 2.0), new Tuple2<>(2, 3.0))); assertArrayEquals(new double[]{2.0, 0.0, 3.0}, v.toArray(), 0.0); } }
@Test public void sparseArrayConstruction() { @SuppressWarnings("unchecked") Vector v = Vectors.sparse(3, Arrays.asList( new Tuple2<>(0, 2.0), new Tuple2<>(2, 3.0))); assertArrayEquals(new double[]{2.0, 0.0, 3.0}, v.toArray(), 0.0); } }
@Test public void sparseArrayConstruction() { @SuppressWarnings("unchecked") Vector v = Vectors.sparse(3, Arrays.asList( new Tuple2<>(0, 2.0), new Tuple2<>(2, 3.0))); assertArrayEquals(new double[]{2.0, 0.0, 3.0}, v.toArray(), 0.0); } }
public LabeledPoint call(Row r) { Integer val = r.getInt(0); SparseVector vec = (SparseVector)r.get(1); Vector features = Vectors.sparse(vec.size(), vec.indices(), vec.values()); return new LabeledPoint(val, features); } });
public LabeledPoint call(Row r) { Integer val = r.getInt(0); SparseVector vec = (SparseVector) r.get(1); Vector features = Vectors.sparse(vec.size(), vec.indices(), vec.values()); return new LabeledPoint(val, features); } }).cache();
Vector testData = Vectors.sparse(sp.size(), sp.indices(), sp.values()); return linModel.predict(testData.compressed());
@Test public void twoDimensionalData() { JavaRDD<Vector> points = jsc.parallelize(Arrays.asList( Vectors.dense(4, -1), Vectors.dense(4, 1), Vectors.sparse(2, new int[]{0}, new double[]{1.0}) ), 2); BisectingKMeans bkm = new BisectingKMeans() .setK(4) .setMaxIterations(2) .setSeed(1L); BisectingKMeansModel model = bkm.run(points); Assert.assertEquals(3, model.k()); Assert.assertArrayEquals(new double[]{3.0, 0.0}, model.root().center().toArray(), 1e-12); for (ClusteringTreeNode child : model.root().children()) { double[] center = child.center().toArray(); if (center[0] > 2) { Assert.assertEquals(2, child.size()); Assert.assertArrayEquals(new double[]{4.0, 0.0}, center, 1e-12); } else { Assert.assertEquals(1, child.size()); Assert.assertArrayEquals(new double[]{1.0, 0.0}, center, 1e-12); } } } }
@Test public void twoDimensionalData() { JavaRDD<Vector> points = jsc.parallelize(Arrays.asList( Vectors.dense(4, -1), Vectors.dense(4, 1), Vectors.sparse(2, new int[]{0}, new double[]{1.0}) ), 2); BisectingKMeans bkm = new BisectingKMeans() .setK(4) .setMaxIterations(2) .setSeed(1L); BisectingKMeansModel model = bkm.run(points); Assert.assertEquals(3, model.k()); Assert.assertArrayEquals(new double[]{3.0, 0.0}, model.root().center().toArray(), 1e-12); for (ClusteringTreeNode child : model.root().children()) { double[] center = child.center().toArray(); if (center[0] > 2) { Assert.assertEquals(2, child.size()); Assert.assertArrayEquals(new double[]{4.0, 0.0}, center, 1e-12); } else { Assert.assertEquals(1, child.size()); Assert.assertArrayEquals(new double[]{1.0, 0.0}, center, 1e-12); } } } }
@Test public void twoDimensionalData() { JavaRDD<Vector> points = jsc.parallelize(Arrays.asList( Vectors.dense(4, -1), Vectors.dense(4, 1), Vectors.sparse(2, new int[]{0}, new double[]{1.0}) ), 2); BisectingKMeans bkm = new BisectingKMeans() .setK(4) .setMaxIterations(2) .setSeed(1L); BisectingKMeansModel model = bkm.run(points); Assert.assertEquals(3, model.k()); Assert.assertArrayEquals(new double[]{3.0, 0.0}, model.root().center().toArray(), 1e-12); for (ClusteringTreeNode child : model.root().children()) { double[] center = child.center().toArray(); if (center[0] > 2) { Assert.assertEquals(2, child.size()); Assert.assertArrayEquals(new double[]{4.0, 0.0}, center, 1e-12); } else { Assert.assertEquals(1, child.size()); Assert.assertArrayEquals(new double[]{1.0, 0.0}, center, 1e-12); } } } }
@Test public void diagonalMatrixConstruction() { Vector v = Vectors.dense(1.0, 0.0, 2.0); Vector sv = Vectors.sparse(3, new int[]{0, 2}, new double[]{1.0, 2.0}); Matrix m = Matrices.diag(v); Matrix sm = Matrices.diag(sv); DenseMatrix d = DenseMatrix.diag(v); DenseMatrix sd = DenseMatrix.diag(sv); SparseMatrix s = SparseMatrix.spdiag(v); SparseMatrix ss = SparseMatrix.spdiag(sv); assertArrayEquals(m.toArray(), sm.toArray(), 0.0); assertArrayEquals(d.toArray(), sm.toArray(), 0.0); assertArrayEquals(d.toArray(), sd.toArray(), 0.0); assertArrayEquals(sd.toArray(), s.toArray(), 0.0); assertArrayEquals(s.toArray(), ss.toArray(), 0.0); assertArrayEquals(s.values(), ss.values(), 0.0); assertEquals(2, s.values().length); assertEquals(2, ss.values().length); assertEquals(4, s.colPtrs().length); assertEquals(4, ss.colPtrs().length); }
@Test public void diagonalMatrixConstruction() { Vector v = Vectors.dense(1.0, 0.0, 2.0); Vector sv = Vectors.sparse(3, new int[]{0, 2}, new double[]{1.0, 2.0}); Matrix m = Matrices.diag(v); Matrix sm = Matrices.diag(sv); DenseMatrix d = DenseMatrix.diag(v); DenseMatrix sd = DenseMatrix.diag(sv); SparseMatrix s = SparseMatrix.spdiag(v); SparseMatrix ss = SparseMatrix.spdiag(sv); assertArrayEquals(m.toArray(), sm.toArray(), 0.0); assertArrayEquals(d.toArray(), sm.toArray(), 0.0); assertArrayEquals(d.toArray(), sd.toArray(), 0.0); assertArrayEquals(sd.toArray(), s.toArray(), 0.0); assertArrayEquals(s.toArray(), ss.toArray(), 0.0); assertArrayEquals(s.values(), ss.values(), 0.0); assertEquals(2, s.values().length); assertEquals(2, ss.values().length); assertEquals(4, s.colPtrs().length); assertEquals(4, ss.colPtrs().length); }
@Test public void diagonalMatrixConstruction() { Vector v = Vectors.dense(1.0, 0.0, 2.0); Vector sv = Vectors.sparse(3, new int[]{0, 2}, new double[]{1.0, 2.0}); Matrix m = Matrices.diag(v); Matrix sm = Matrices.diag(sv); DenseMatrix d = DenseMatrix.diag(v); DenseMatrix sd = DenseMatrix.diag(sv); SparseMatrix s = SparseMatrix.spdiag(v); SparseMatrix ss = SparseMatrix.spdiag(sv); assertArrayEquals(m.toArray(), sm.toArray(), 0.0); assertArrayEquals(d.toArray(), sm.toArray(), 0.0); assertArrayEquals(d.toArray(), sd.toArray(), 0.0); assertArrayEquals(sd.toArray(), s.toArray(), 0.0); assertArrayEquals(s.toArray(), ss.toArray(), 0.0); assertArrayEquals(s.values(), ss.values(), 0.0); assertEquals(2, s.values().length); assertEquals(2, ss.values().length); assertEquals(4, s.colPtrs().length); assertEquals(4, ss.colPtrs().length); }